Azure Terraform 故障排查

Error: creating Kubernetes Cluster ... unexpected status 400
"code": "K8sVersionNotSupported"
"message": "Managed cluster ... is on version 1.32.x, which is only available for Long-Term Support (LTS).
If you intend to onboard to LTS, please ensure the cluster is in Premium tier ..."

az aks get-versions --location eastus -o table
# Versions with KubernetesOfficial in SupportPlan column work on Standard tier

Warning  FailedScheduling     pod/langsmith-backend-xxx  0/1 nodes are available: 1 Too many pods.
Normal   NotTriggerScaleUp    pod/langsmith-backend-xxx  pod didn't trigger scale-up: 2 in backoff after failed scale-up

Error: creating temporary Agent Pool ... "code": "ErrCode_InsufficientVCPUQuota",
"message": "Insufficient vcpu quota requested 8, remaining 2 for family standardDSv3Family for region eastus."

# Azure portal usually auto-approves within minutes:
# Portal → Subscriptions → <sub> → Usage + Quotas → search "DSv3" → eastus → Request increase → 32

# Or via CLI
az quota update \
  --resource-name "standardDSv3Family" \
  --scope /subscriptions/<sub-id>/providers/Microsoft.Compute/locations/eastus \
  --limit-object value=32 limit-type=Independent \
  --resource-type dedicated

az vm list-usage --location eastus --query "[?contains(name.value,'DSv3')]" -o table

az aks mesh get-revisions --location eastus -o table

Error: updating Key Vault "langsmith-kv-dz":
once Purge Protection has been Enabled it's not possible to disable it

keyvault_purge_protection = true

# 1. Remove KV from Terraform state (does not delete from Azure)
terraform -chdir=infra state rm module.keyvault.azurerm_key_vault.langsmith

# 2. Permanently purge the soft-deleted KV (irreversible)
az keyvault purge --name langsmith-kv<identifier> --location eastus

# 3. Re-apply
make apply

Error: a resource with the ID "https://langsmith-kv-<id>.vault.azure.net/secrets/.../..."
already exists - to be managed via Terraform this resource needs to be imported into the State.

terraform import \
  'module.keyvault.azurerm_key_vault_secret.deployments_encryption_key[0]' \
  "$(az keyvault secret show --vault-name langsmith-kv<id> --name langsmith-deployments-encryption-key --query id -o tsv)"

terraform import \
  'module.keyvault.azurerm_key_vault_secret.agent_builder_encryption_key[0]' \
  "$(az keyvault secret show --vault-name langsmith-kv<id> --name langsmith-agent-builder-encryption-key --query id -o tsv)"

terraform import \
  'module.keyvault.azurerm_key_vault_secret.insights_encryption_key[0]' \
  "$(az keyvault secret show --vault-name langsmith-kv<id> --name langsmith-insights-encryption-key --query id -o tsv)"

terraform apply

kubectl annotate svc ingress-nginx-controller -n ingress-nginx \
  service.beta.kubernetes.io/azure-dns-label-name=<dns_label> \
  --overwrite

# Wait 1-2 minutes, verify DNS resolves
nslookup <dns_label>.eastus.cloudapp.azure.com

# Delete the stuck cert to trigger re-issue
kubectl delete certificate langsmith-tls -n langsmith

kubectl apply -f - <<EOF
apiVersion: cert-manager.io/v1
kind: ClusterIssuer
metadata:
  name: letsencrypt-prod
spec:
  acme:
    server: https://acme-v02.api.letsencrypt.org/directory
    email: you@example.com
    privateKeySecretRef:
      name: letsencrypt-prod-account-key
    solvers:
    - http01:
        ingress:
          ingressClassName: nginx   # use "istio" with istio-addon or istio
EOF

kubectl delete certificate langsmith-tls -n langsmith

terraform apply
kubectl rollout restart deployment -n langsmith

kubectl delete secret langsmith-config-secret -n langsmith
make k8s-secrets   # recreates with correct key names
make deploy

config:
  deployment:
    enabled: true
    url: "https://langsmith-demo.eastus.cloudapp.azure.com"   # 必须包含 https://
    tlsEnabled: true   # 当 tls_certificate_source = letsencrypt 或 dns01 时必须为 true

cat > helm/values/langsmith-values-insights.yaml << 'EOF'
insights:
  enabled: true
EOF
make deploy

kubectl rollout restart deployment langsmith-frontend -n langsmith
kubectl exec -n langsmith deploy/langsmith-frontend -- env | grep POLLY
# expect: VITE_POLLY_DEPLOYMENT_URL=https://<hostname>/lgp/smith-polly-<hash>

config:
  deployment:
    enabled: true          # 必填；仅有 url 是不够的
    url: "https://<your-hostname>"

panic: blob-storage health-check failed: get container properties failed:
DefaultAzureCredential: failed to acquire a token.
WorkloadIdentityCredential authentication failed.
  AADSTS700213: No matching federated identity record found for presented assertion subject
  'system:serviceaccount:langsmith:langsmith-<service>'

service_accounts_for_workload_identity = [
  "${var.langsmith_release_name}-backend",
  "${var.langsmith_release_name}-platform-backend",
  "${var.langsmith_release_name}-queue",
  "${var.langsmith_release_name}-ingest-queue",
  "${var.langsmith_release_name}-host-backend",                 # LangSmith Deployment 附加组件
  "${var.langsmith_release_name}-listener",                     # LangSmith Deployment 附加组件
  "${var.langsmith_release_name}-agent-builder-tool-server",    # Agent Builder 附加组件
  "${var.langsmith_release_name}-agent-builder-trigger-server", # Agent Builder 附加组件
]

terraform apply -target=module.aks
kubectl rollout restart deployment/langsmith-<service> -n langsmith

1. make uninstall   ← Helm + namespace
2. make destroy     ← Azure infra (needs tfstate + tfvars)
3. make clean       ← local secrets and generated files (LAST)

az group delete --name langsmith-rg<identifier> --yes --no-wait
az group show --name langsmith-rg<identifier> 2>&1 | grep -E "provisioningState|ResourceGroupNotFound"

make uninstall
kubectl delete namespace langsmith --timeout=60s
make destroy

kubectl delete hpa langsmith-listener langsmith-host-backend -n langsmith 2>/dev/null || true
kubectl scale deployment langsmith-listener -n langsmith --replicas=1
kubectl scale deployment langsmith-host-backend -n langsmith --replicas=1
make deploy

az aks update --name <CLUSTER> --resource-group <RG> --yes
kubectl delete pod -n kube-system -l app=ingress-azure

AGIC_OID=$(az aks show -g <RG> -n <CLUSTER> \
  --query "addonProfiles.ingressApplicationGateway.identity.objectId" -o tsv)
VNET_ID=$(az network vnet show -g <RG> -n <VNET> --query id -o tsv)

az role assignment create --role "Network Contributor" --scope "$VNET_ID" \
  --assignee-object-id "$AGIC_OID" --assignee-principal-type ServicePrincipal

kubectl rollout restart deployment/ingress-appgw-deployment -n kube-system

kubectl get certificate langsmith-tls -n langsmith   # verify cert is ready
kubectl annotate ingress langsmith-ingress -n langsmith touch="$(date +%s)" --overwrite

kubectl get crd | grep "istio.io" | awk '{print $1}' | xargs kubectl delete crd
terraform apply

az aks get-credentials --name <cluster> --resource-group <rg>
kubectl config current-context
kubectl get nodes -o wide

kubectl get pods -n langsmith
kubectl get pods -n langsmith -w
kubectl describe pod <pod-name> -n langsmith
kubectl logs <pod-name> -n langsmith --tail=100 -f
kubectl logs <pod-name> -n langsmith --previous --tail=50

kubectl get ingress -n langsmith
kubectl get svc ingress-nginx-controller -n ingress-nginx
kubectl get certificate -n langsmith
kubectl get challenges -n langsmith
kubectl get clusterissuer

kubectl get serviceaccount langsmith-ksa -n langsmith \
  -o jsonpath='{.metadata.annotations.azure\.workload\.identity/client-id}'

kubectl get pod <pod> -n langsmith \
  -o jsonpath='{.metadata.labels.azure\.workload\.identity/use}'

helm status langsmith -n langsmith
helm history langsmith -n langsmith
helm get values langsmith -n langsmith

kubectl get pods -n langsmith | grep -E "host-backend|listener|operator"
kubectl get lgp -n langsmith
kubectl get crd | grep langchain

./infra/scripts/manage-keyvault.sh list
./infra/scripts/manage-keyvault.sh validate
./infra/scripts/manage-keyvault.sh diff

kubectl get secrets -n langsmith
kubectl get secret langsmith-config-secret -n langsmith -o jsonpath='{.data}' | jq 'keys'

make status         # 10-section automated check
make status-quick   # skip Key Vault + K8s secret queries