ci(publish): restore Soda PyPI upload, fix --skip-existing on devpi

Reverses the "drop Soda PyPI" half of this branch — soda-core clean
releases are still needed on pypi.cloud.soda.io. The original failure was
not that the registry is wrong, but that #2758 passed --skip-existing to
every leg, and pypi.cloud.soda.io (devpi) rejects that flag:

  UnsupportedConfiguration: 'https://pypi.cloud.soda.io' does not have
  support for the following features: --skip-existing

Fix per-registry instead of removing the leg:
  - public PyPI        -> keeps `twine upload --skip-existing` (supported)
  - pypi.cloud.soda.io -> upload plain; capture output and treat an
                          "already exists" / 409 rejection as success, so
                          re-runs and backfills stay idempotent

Keeps this branch's single-approval gate (one approval email per run, not
one per matrix leg). Restores the AWS secrets fetch the Soda leg needs.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: paulteehan

.github/workflows/publish.yaml

@@ -1,7 +1,7 @@
 ---
 name: Publish to PyPI
 
-# Publishes an ALREADY-TAGGED release to public PyPI.
+# Publishes an ALREADY-TAGGED release to Soda PyPI and public PyPI.
 #
 # Why this exists, separate from release.yaml:
 #   release.yaml ("one-button release") resolves a version, tags it, builds,
@@ -28,8 +28,13 @@ name: Publish to PyPI
 #                           (e.g. v4.14.0, or the 4.8.0-4.13.0 backlog); a tag
 #                           pushed before this workflow existed won't re-fire.
 #
-# twine runs with --skip-existing, so re-runs and backfills are safe and the
-# automatic tag path does not fail when public PyPI already has the version.
+# Idempotency (re-runs / backfills must not fail on an already-published
+# version) is handled per-registry, because the two indexes differ:
+#   public PyPI        -> supports `twine upload --skip-existing`.
+#   pypi.cloud.soda.io -> devpi index; does NOT support --skip-existing
+#                         (`UnsupportedConfiguration: ... --skip-existing`).
+#                         So we upload plain and treat an "already exists"
+#                         rejection as success.
 #
 # Approval: only the `approve` job references the production-release
 # environment, so reviewers get ONE approval request per run rather than one
@@ -122,6 +127,17 @@ jobs:
       - name: Set up UV
         uses: astral-sh/setup-uv@e58605a9b6da7c637471fab8847a5e5a6b8df081  # v5
 
+      - name: Get external secrets
+        uses: aws-actions/aws-secretsmanager-get-secrets@a9a7eb4e2f2871d30dc5b892576fde60a2ecc802  # v2.0.10
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_BUILD_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_BUILD_SECRET_ACCESS_KEY }}
+          AWS_REGION: ${{ secrets.AWS_BUILD_DEFAULT_REGION }}
+        with:
+          secret-ids: |
+            PYPI_USERNAME,/soda/github/common/SODA_PYPI_CLOUD_WRITE_USERNAME
+            PYPI_PASSWORD,/soda/github/common/SODA_PYPI_CLOUD_WRITE_PASSWORD
+
       - name: Build ${{ matrix.module }}
         run: |
           uv venv .venv
@@ -152,6 +168,39 @@ jobs:
           echo "Staggering upload by ${DELAY}s..."
           sleep "$DELAY"
 
+      - name: Publish to Soda PyPI
+        id: soda_pypi
+        if: ${{ !inputs.dry_run }}
+        env:
+          TWINE_REPOSITORY_URL: ${{ vars.CLOUD_PYPI_REPOSITORY }}
+          TWINE_USERNAME: ${{ env.PYPI_USERNAME }}
+          TWINE_PASSWORD: ${{ env.PYPI_PASSWORD }}
+        run: |
+          source .venv/bin/activate
+          # pypi.cloud.soda.io (devpi) does NOT support --skip-existing, so we
+          # upload plain and treat an "already exists" rejection as success to
+          # keep re-runs / backfills idempotent.
+          RESULT="failure"
+          for attempt in 1 2 3; do
+            OUT="$(twine upload "${{ matrix.module }}"/dist/* 2>&1)" && {
+              echo "$OUT"; RESULT="success"
+              echo "Soda PyPI upload succeeded on attempt $attempt"; break
+            }
+            echo "$OUT"
+            if echo "$OUT" | grep -qiE "already (exists|been uploaded)|this filename has already been used|409 (conflict|client error)"; then
+              RESULT="success"
+              echo "Soda PyPI already has this version — treating as success."; break
+            fi
+            if [ "$attempt" -lt 3 ]; then
+              DELAY=$((attempt * 15))
+              echo "::warning::Soda PyPI upload failed (attempt $attempt/3), retrying in ${DELAY}s..."
+              sleep "$DELAY"
+            else
+              echo "::error::Soda PyPI upload failed after 3 attempts"
+            fi
+          done
+          echo "result=$RESULT" >> "$GITHUB_OUTPUT"
+
       - name: Publish to public PyPI
         id: public_pypi
         if: ${{ !inputs.dry_run }}
@@ -179,8 +228,9 @@ jobs:
         if: ${{ always() && !inputs.dry_run }}
         run: |
           echo "### ${{ matrix.module }} @ ${{ needs.resolve.outputs.tag }}" >> "$GITHUB_STEP_SUMMARY"
+          echo "- Soda PyPI: ${{ steps.soda_pypi.outputs.result }}" >> "$GITHUB_STEP_SUMMARY"
           echo "- Public PyPI: ${{ steps.public_pypi.outputs.result }}" >> "$GITHUB_STEP_SUMMARY"
-          if [ "${{ steps.public_pypi.outputs.result }}" != "success" ]; then
-            echo "::error::${{ matrix.module }} did not publish to public PyPI — see logs."
+          if [ "${{ steps.soda_pypi.outputs.result }}" != "success" ] || [ "${{ steps.public_pypi.outputs.result }}" != "success" ]; then
+            echo "::error::${{ matrix.module }} did not publish cleanly to both registries — see logs."
             exit 1
           fi