From b195cd01713c0a17c65c065a55447f2b186fdb5a Mon Sep 17 00:00:00 2001 From: mik-tf Date: Thu, 6 Nov 2025 20:53:56 -0500 Subject: [PATCH] docs: Add comprehensive troubleshooting guide for nginx-nodeport with improved test script --- examples/nginx-nodeport/TROUBLESHOOTING.md | 347 ++++++++++++++++++ examples/nginx-nodeport/test-nodeport-ipv6.sh | 54 ++- 2 files changed, 386 insertions(+), 15 deletions(-) create mode 100644 examples/nginx-nodeport/TROUBLESHOOTING.md diff --git a/examples/nginx-nodeport/TROUBLESHOOTING.md b/examples/nginx-nodeport/TROUBLESHOOTING.md new file mode 100644 index 0000000..36b71d1 --- /dev/null +++ b/examples/nginx-nodeport/TROUBLESHOOTING.md @@ -0,0 +1,347 @@ +# nginx-nodeport Troubleshooting Guide + +## Common Issues and Solutions + +### Issue: External IPv6 Connectivity Test Failed + +**Symptoms:** +``` +⚠️ External Mycelium IPv6 connectivity test failed +ℹ️ This might be due to: +ℹ️ • Firewall or network policies blocking port 30091 +ℹ️ • Pod not running on this specific node +ℹ️ • Mycelium network not properly configured +``` + +**Root Causes & Solutions:** + +#### 1. Mycelium Not Running on Your Machine ⭐ Most Common + +**Problem:** You're trying to access from a machine without Mycelium installed/running. + +**Solution:** Install and start Mycelium on your local machine: + +```bash +# Download Mycelium +wget https://github.com/threefoldtech/mycelium/releases/latest/download/mycelium-private-x86_64-unknown-linux-musl.tar.gz +tar -xzf mycelium-private-x86_64-unknown-linux-musl.tar.gz +sudo chmod +x mycelium-private +sudo mv mycelium-private /usr/local/bin/mycelium + +# Start Mycelium with peers +sudo mycelium --peers tcp://188.40.132.242:9651 tcp://136.243.47.186:9651 tcp://185.69.166.7:9651 +``` + +**After starting Mycelium, test connectivity:** +```bash +# Your machine should now have a Mycelium IPv6 address +ip addr show | grep mycelium + +# Test connection +NODE_IPV6="552:5984:2d97:72dc:ff0f:39ef:6ec:a48c" # Replace with your node's IP +curl -6 "http://[$NODE_IPV6]:30091/" +``` + +--- + +#### 2. Testing from Wrong Location + +**Problem:** Testing from within cluster or wrong network segment. + +**Solution:** Test from a machine WITH Mycelium running: + +```bash +# ✅ CORRECT: Test from machine with Mycelium +your-laptop$ curl -6 "http://[node-mycelium-ipv6]:30091/" + +# ❌ WRONG: Test from within cluster (use localhost instead) +kubectl exec pod/nginx-nodeport-xxx -- curl localhost:8080 +``` + +--- + +#### 3. Firewall Blocking NodePort + +**Problem:** Firewall rules blocking port 30091. + +**Check firewall:** +```bash +# On the worker node +sudo iptables -L -n | grep 30091 +sudo firewall-cmd --list-ports # if using firewalld +``` + +**Solution:** Open port 30091 on worker nodes: +```bash +# Using iptables +sudo iptables -I INPUT -p tcp --dport 30091 -j ACCEPT + +# Using firewalld +sudo firewall-cmd --permanent --add-port=30091/tcp +sudo firewall-cmd --reload +``` + +--- + +#### 4. externalTrafficPolicy: Local Issue + +**Problem:** With `externalTrafficPolicy: Local`, service is only accessible on nodes where pods are running. + +**Understanding:** +- ✅ Pod on node A → Access via node A's IP works +- ❌ Pod on node A → Access via node B's IP fails + +**Solution:** Always use the IP of the node where your pod is running: + +```bash +# Find which node has your pod +POD_NODE=$(kubectl get pods -l app=nginx-nodeport -o jsonpath='{.items[0].spec.nodeName}') +echo "Pod is on: $POD_NODE" + +# Get THAT node's Mycelium IPv6 +NODE_IPV6=$(kubectl get node "$POD_NODE" -o jsonpath='{range .status.addresses[?(@.type=="InternalIP")]}{.address}{"\n"}{end}' | grep ':' | head -1) +echo "Use this IP: $NODE_IPV6" + +# Test +curl -6 "http://[$NODE_IPV6]:30091/" +``` + +--- + +#### 5. IPv6 vs IPv4 Confusion + +**Problem:** Getting both IPv4 and IPv6 addresses, causing malformed URLs. + +**Symptoms:** +``` +curl: (3) bad range in URL position 9: +http://[10.20.2.2 51d:3596:6cc3:81e7:ff0f:d546:3737:4c8c]:30091/ +``` + +**Solution:** Filter for IPv6 only: +```bash +# Correct way to get IPv6 only +kubectl get nodes -o jsonpath='{range .items[*]}{range .status.addresses[?(@.type=="InternalIP")]}{.address}{"\n"}{end}{end}' | grep -E '^[0-9a-f]+:[0-9a-f]+:' + +# NOT this (gets both IPv4 and IPv6) +kubectl get nodes -o jsonpath='{.items[0].status.addresses[?(@.type=="InternalIP")].address}' +``` + +--- + +### Issue: Pod Not Accessible + +**Check pod status:** +```bash +kubectl get pods -l app=nginx-nodeport +kubectl describe pod -l app=nginx-nodeport +kubectl logs -l app=nginx-nodeport +``` + +**Test internal connectivity:** +```bash +POD_NAME=$(kubectl get pods -l app=nginx-nodeport -o name | head -1) + +# Test from within pod +kubectl exec $POD_NAME -- curl -s http://localhost:8080/ + +# Test health endpoint +kubectl exec $POD_NAME -- curl -s http://localhost:8080/health +``` + +--- + +### Issue: Service Not Working + +**Check service configuration:** +```bash +# Verify service type +kubectl get svc nginx-nodeport-service -o jsonpath='{.spec.type}' +# Should output: NodePort + +# Check NodePort assignment +kubectl get svc nginx-nodeport-service -o jsonpath='{.spec.ports[0].nodePort}' +# Should output: 30091 + +# Check endpoints +kubectl get endpoints nginx-nodeport-service +# Should show pod IP and port +``` + +--- + +### Issue: Wrong Content Displayed + +**Update ConfigMap:** +```bash +# Regenerate content with current IPs +./update-content.sh + +# Restart deployment to pick up changes +kubectl rollout restart deployment/nginx-nodeport + +# Wait for rollout +kubectl rollout status deployment/nginx-nodeport +``` + +--- + +## Testing Checklist + +Run through this checklist to verify everything: + +- [ ] **Mycelium Running:** `ip addr show | grep mycelium` +- [ ] **Pod Running:** `kubectl get pods -l app=nginx-nodeport` +- [ ] **Service Exists:** `kubectl get svc nginx-nodeport-service` +- [ ] **NodePort 30091:** `kubectl get svc nginx-nodeport-service -o jsonpath='{.spec.ports[0].nodePort}'` +- [ ] **Pod Node Located:** `kubectl get pods -l app=nginx-nodeport -o wide` +- [ ] **Node IPv6 Found:** `kubectl get node -o jsonpath='{.status.addresses[?(@.type=="InternalIP")].address}' | grep ':'` +- [ ] **Internal Access:** `kubectl exec -- curl -s http://localhost:8080/` +- [ ] **External Access:** `curl -6 "http://[node-ipv6]:30091/"` + +--- + +## Quick Validation Commands + +```bash +# Complete test sequence +cd myceliumcloud-examples/examples/nginx-nodeport + +# 1. Run automated tests +./test-nodeport-ipv6.sh + +# 2. Find your pod's node +POD_NODE=$(kubectl get pods -l app=nginx-nodeport -o jsonpath='{.items[0].spec.nodeName}') +echo "Pod is on node: $POD_NODE" + +# 3. Get that node's Mycelium IPv6 +NODE_IPV6=$(kubectl get node "$POD_NODE" -o jsonpath='{range .status.addresses[?(@.type=="InternalIP")]}{.address}{"\n"}{end}' | grep -E '^[0-9a-f]+:[0-9a-f]+:' | head -1) +echo "Node Mycelium IPv6: $NODE_IPV6" + +# 4. Test from machine with Mycelium +curl -6 "http://[$NODE_IPV6]:30091/" +``` + +--- + +## Expected Behavior + +### ✅ Working Correctly + +When everything is working, you should see: + +1. **Test Script Output:** +``` +✅ nginx-nodeport pods are ready +✅ nginx configuration is valid +✅ Health endpoint is working +✅ NodePort service is configured correctly +✅ NodePort is correctly set to 30091 +``` + +2. **External Access:** +```bash +$ curl -6 "http://[552:5984:2d97:72dc:ff0f:39ef:6ec:a48c]:30091/" + + + + Mycelium Cloud - Nginx NodePort Website +... +``` + +3. **Browser Access:** + - Navigate to: `http://[node-mycelium-ipv6]:30091/` + - Should see: Styled website with "NODEPORT SECURE" badge + +--- + +## Advanced Debugging + +### Check kube-proxy + +```bash +# On worker node +sudo iptables-save | grep 30091 +``` + +### View Service Routing + +```bash +# Check service details +kubectl describe svc nginx-nodeport-service + +# Check endpoints +kubectl get endpoints nginx-nodeport-service -o yaml +``` + +### Test from Another Pod + +```bash +# Create test pod +kubectl run test-pod --image=nginx:alpine --rm -it -- sh + +# From inside test pod +wget -O- http://nginx-nodeport-service:8080/ +# Should work (internal cluster access) + +# Try NodePort (won't work from inside cluster) +wget -O- http://[node-ip]:30091/ +# May not work depending on network policies +``` + +--- + +## Still Having Issues? + +If none of the above solutions work: + +1. **Check Logs:** + ```bash + kubectl logs -l app=nginx-nodeport --tail=50 + ``` + +2. **Describe Resources:** + ```bash + kubectl describe deployment nginx-nodeport + kubectl describe service nginx-nodeport-service + kubectl describe pod -l app=nginx-nodeport + ``` + +3. **Verify Network:** + ```bash + # Check pod can reach internet + kubectl exec -- ping -c 3 8.8.8.8 + + # Check pod networking + kubectl exec -- ip addr + kubectl exec -- ip route + ``` + +4. **Compare with hostNetwork:** + - The [`nginx-mycelium`](../nginx-mycelium/) example uses hostNetwork + - If that works but NodePort doesn't, it's likely a service/networking issue + - If neither works, check Mycelium connectivity + +--- + +## Common Questions + +**Q: Why can't I access from any node's IP?** +A: With `externalTrafficPolicy: Local`, you can only access via the node where the pod is running. This preserves source IP but limits accessibility. + +**Q: Should I change to `externalTrafficPolicy: Cluster`?** +A: You can, but you'll lose source IP preservation. Decision depends on your logging/security needs. + +**Q: How is this different from hostNetwork?** +A: NodePort provides network isolation and standard Kubernetes patterns. HostNetwork gives direct host access (less secure). See [compare-approaches.md](compare-approaches.md). + +**Q: Why port 30091 and not 80?** +A: Kubernetes NodePorts use range 30000-32767. Port 80 requires LoadBalancer or Ingress type services. + +--- + +**Last Updated:** 2025-01-07 +**Related Docs:** +- [nginx-nodeport.md](nginx-nodeport.md) - Complete guide +- [compare-approaches.md](compare-approaches.md) - Security comparison +- [../nginx-variants.md](../nginx-variants.md) - All nginx variants \ No newline at end of file diff --git a/examples/nginx-nodeport/test-nodeport-ipv6.sh b/examples/nginx-nodeport/test-nodeport-ipv6.sh index 827cdf8..1b9fda2 100755 --- a/examples/nginx-nodeport/test-nodeport-ipv6.sh +++ b/examples/nginx-nodeport/test-nodeport-ipv6.sh @@ -67,9 +67,11 @@ else exit 1 fi -# Get pod information +# Get pod information and node it's running on POD_NAME=$(kubectl get pods -l app=nginx-nodeport -o name | head -1) +POD_NODE=$(kubectl get pods -l app=nginx-nodeport -o jsonpath='{.items[0].spec.nodeName}') print_info "Testing pod: $POD_NAME" +print_info "Pod is running on node: $POD_NODE" # Test nginx configuration print_info "Testing nginx configuration..." @@ -118,30 +120,48 @@ else fi print_info "NodePort: $NODEPORT" -# Get node IPv6 address -print_info "Getting node IPv6 address..." -NODE_IPV6=$(kubectl get nodes -o jsonpath='{.items[0].status.addresses[?(@.type=="InternalIP")].address}' 2>/dev/null || echo "") +# Get the Mycelium IPv6 address of the SPECIFIC node where the pod is running +# This is critical with externalTrafficPolicy: Local +print_info "Getting Mycelium IPv6 address of node where pod is running..." + +# Get Mycelium IPv6 of the specific node where our pod is running +NODE_IPV6=$(kubectl get node "$POD_NODE" -o jsonpath='{range .status.addresses[?(@.type=="InternalIP")]}{.address}{"\n"}{end}' 2>/dev/null | grep -E '^[0-9a-f]+:[0-9a-f]+:[0-9a-f]+:[0-9a-f]+:[0-9a-f]+:[0-9a-f]+:[0-9a-f]+:[0-9a-f]+$' | head -1) if [ -z "$NODE_IPV6" ]; then - print_warning "Could not get node IPv6 address automatically" + print_warning "Could not get Mycelium IPv6 address for node $POD_NODE" print_info "Please manually find your node IPv6 address with:" - echo " kubectl get nodes -o wide" + echo " kubectl get node $POD_NODE -o jsonpath='{range .status.addresses[?(@.type==\"InternalIP\")]}{.address}{\"\n\"}{end}' | grep ':'" else - print_status "Node IPv6 address: $NODE_IPV6" + print_status "Pod's node ($POD_NODE) Mycelium IPv6: $NODE_IPV6" + print_info "⚠️ NOTE: With externalTrafficPolicy: Local, service is only accessible on THIS node's IP" # Test external connectivity - print_info "Testing external IPv6 connectivity..." + print_info "Testing external Mycelium IPv6 connectivity..." # Test with IPv6 if command -v curl &> /dev/null; then - if curl -6 -s -m 10 "http://[$NODE_IPV6]:$NODEPORT/" &> /dev/null; then - print_status "External IPv6 connectivity is working!" - print_info "Your website is accessible at: http://[$NODE_IPV6]:$NODEPORT/" + print_info "Testing: curl -6 \"http://[$NODE_IPV6]:$NODEPORT/\"" + if curl -6 -s -m 10 "http://[$NODE_IPV6]:$NODEPORT/" > /tmp/nodeport_test.html 2>/dev/null; then + if [ -s /tmp/nodeport_test.html ]; then + print_status "External Mycelium IPv6 connectivity is working!" + print_info "Your website is accessible at: http://[$NODE_IPV6]:$NODEPORT/" + print_info "Content preview:" + head -3 /tmp/nodeport_test.html | sed 's/^/ /' + rm -f /tmp/nodeport_test.html + else + print_warning "Connected but received no content" + fi else - print_warning "External IPv6 connectivity test failed" - print_info "This might be due to firewall or network policies" - print_info "Website should still be accessible from within the cluster" + print_warning "External Mycelium IPv6 connectivity test failed" + print_info "This might be due to:" + print_info " • Firewall or network policies blocking port $NODEPORT" + print_info " • Pod not running on this specific node" + print_info " • Mycelium network not properly configured" + print_info "" + print_info "Try testing from within the cluster:" + echo " kubectl exec $POD_NAME -- curl -s http://localhost:8080/" fi + rm -f /tmp/nodeport_test.html else print_info "curl not available, skipping external connectivity test" fi @@ -198,8 +218,12 @@ if [ ! -z "$NODE_IPV6" ]; then echo " • External URL: http://[$NODE_IPV6]:$NODEPORT/" echo " • Health check: http://[$NODE_IPV6]:$NODEPORT/health" echo " • Internal test: kubectl exec $POD_NAME -- curl -s http://localhost:8080/" + echo "" + echo " 💡 All worker node Mycelium IPv6 addresses:" + kubectl get nodes -o jsonpath='{range .items[*]}{range .status.addresses[?(@.type=="InternalIP")]}{.address}{"\n"}{end}{end}' | grep -E '^[0-9a-f]+:[0-9a-f]+:[0-9a-f]+:[0-9a-f]+:[0-9a-f]+:[0-9a-f]+:[0-9a-f]+:[0-9a-f]+$' | nl -v 1 -w 5 -s '. ' | sed 's/^/ /' else - echo " • Please get your node IPv6 address: kubectl get nodes -o wide" + echo " • Get your node Mycelium IPv6 addresses:" + echo " kubectl get nodes -o jsonpath='{range .items[*]}{range .status.addresses[?(@.type==\"InternalIP\")]}{.address}{\"\n\"}{end}{end}' | grep ':'" echo " • Access URL: http://[YOUR-NODE-IPV6]:$NODEPORT/" fi echo ""