Enable detached catalog signing for PyManager index files

windows-release/azure-pipelines.yml

@@ -320,6 +320,7 @@ stages:
           BuildToPublish: ${{ parameters.BuildToPublish }}
           DoEmbed: ${{ parameters.DoEmbed }}
           DoFreethreaded: ${{ parameters.DoFreethreaded }}
+          SigningCertificate: ${{ iif(eq(parameters.SigningCertificate, 'Unsigned'), '', parameters.SigningCertificate) }}
     - ${{ if eq(parameters.DoMSI, 'true') }}:
       - template: stage-publish-pythonorg.yml
         parameters:

windows-release/merge-and-upload.py

@@ -20,12 +20,18 @@
 UPLOAD_USER = os.getenv("UPLOAD_USER", "")
 NO_UPLOAD = os.getenv("NO_UPLOAD", "no")[:1].lower() in "yt1"
 LOCAL_INDEX = os.getenv("LOCAL_INDEX", "no")[:1].lower() in "yt1"
+SIGN_COMMAND = os.getenv("SIGN_COMMAND", "")
 
 
 def find_cmd(env, exe):
     cmd = os.getenv(env)
     if cmd:
-        return Path(cmd)
+        cmd = Path(cmd)
+        if not cmd.is_file():
+            raise RuntimeError(
+                f"Could not find {cmd} to perform upload. Incorrect %{env}% setting."
+            )
+        return cmd
     for p in os.getenv("PATH", "").split(";"):
         if p:
             cmd = Path(p) / exe
@@ -40,6 +46,7 @@ def find_cmd(env, exe):
 
 PLINK = find_cmd("PLINK", "plink.exe")
 PSCP = find_cmd("PSCP", "pscp.exe")
+MAKECAT = find_cmd("MAKECAT", "makecat.exe")
 
 
 def _std_args(cmd):
@@ -60,7 +67,9 @@ class RunError(Exception):
     pass
 
 
-def _run(*args):
+def _run(*args, single_cmd=False):
+    if single_cmd:
+        args = args[0]
     with subprocess.Popen(
         args,
         stdout=subprocess.PIPE,
@@ -193,6 +202,43 @@ def calculate_uploads():
         )
 
 
+def sign_json(cat_file, *files):
+    if not MAKECAT:
+        if not UPLOAD_HOST or NO_UPLOAD:
+            print("makecat.exe not found, but not uploading, so skip signing.")
+            return
+        raise RuntimeError("No makecat.exe found")
+    if not SIGN_COMMAND:
+        if not UPLOAD_HOST or NO_UPLOAD:
+            print("No signing command set, but not uploading, so skip signing.")
+            return
+        raise RuntimeError("No SIGN_COMMAND set")
+
+    cat = Path(cat_file).absolute()
+    cdf = cat.with_suffix(".cdf")
+    cdf.parent.mkdir(parents=True, exist_ok=True)
+
+    with open(cdf, "w", encoding="ansi") as f:
+        print("[CatalogHeader]", file=f)
+        print("Name=", cat.name, sep="", file=f)
+        print("ResultDir=", cat.parent, sep="", file=f)
+        print("PublicVersion=0x00000001", file=f)
+        print("CatalogVersion=2", file=f)
+        print("HashAlgorithms=SHA256", file=f)
+        print("EncodingType=", file=f)
+        print(file=f)
+        print("[CatalogFiles]", file=f)
+        for a in map(Path, files):
+            print("<HASH>", a.name, "=", a.absolute(), sep="", file=f)
+
+    _run(MAKECAT, "-v", cdf)
+    if not cat.is_file():
+        raise FileNotFoundError(cat)
+    # Pass as a single arg because the command variable has its own arguments
+    _run(f'{SIGN_COMMAND} "{cat}"', single_cmd=True)
+    cdf.unlink()
+
+
 def remove_and_insert(index, new_installs):
     new = {(i["id"].casefold(), i["sort-version"].casefold()) for i in new_installs}
     to_remove = [
@@ -274,6 +320,7 @@ def find_missing_from_index(url, installs):
         except FileNotFoundError:
             pass
 
+
 print(INDEX_PATH, "mtime =", INDEX_MTIME)
 
 
@@ -284,10 +331,16 @@ def find_missing_from_index(url, installs):
 
 if INDEX_FILE:
     INDEX_FILE = Path(INDEX_FILE).absolute()
+    INDEX_CAT_FILE = INDEX_FILE.with_suffix(".cat")
     INDEX_FILE.parent.mkdir(parents=True, exist_ok=True)
     with open(INDEX_FILE, "w", encoding="utf-8") as f:
         json.dump(index, f)
 
+    sign_json(INDEX_CAT_FILE, INDEX_FILE)
+else:
+    INDEX_CAT_FILE = None
+
+
 if MANIFEST_FILE:
     # Use the sort-version so that the manifest name includes prerelease marks
     MANIFEST_FILE = Path(MANIFEST_FILE).absolute()
@@ -333,6 +386,10 @@ def find_missing_from_index(url, installs):
         print("Uploading", INDEX_FILE, "to", INDEX_URL)
         upload_ssh(INDEX_FILE, INDEX_PATH)
 
+    if INDEX_CAT_FILE:
+        print("Uploading", INDEX_CAT_FILE, "to", f"{INDEX_URL}.cat")
+        upload_ssh(INDEX_CAT_FILE, f"{INDEX_PATH}.cat")
+
     print("Purging", len(UPLOADS), "uploaded files")
     parents = set()
     for i, *_ in UPLOADS:

windows-release/sign-files.yml

@@ -6,33 +6,85 @@ parameters:
   ExtractDir: ''
   SigningCertificate: ''
   ExportCommand: ''
+  ExportLegacyCommand: ''
   ContinueOnError: false
+  InstallTool: true
+  InstallLegacyTool: false
   AzureServiceConnectionName: 'Python Signing'
 
 steps:
-- ${{ if parameters.SigningCertificate }}:
-  - powershell: |
-      # Install sign tool
-      dotnet tool install --global --prerelease sign
-      $signtool = (gcm sign -EA SilentlyContinue).Source
-      if (-not $signtool) {
-        $signtool = (gi "${env:USERPROFILE}\.dotnet\tools\sign.exe").FullName
-      }
-      $signargs = 'code trusted-signing -v Information ' + `
-        '-fd sha256 -t http://timestamp.acs.microsoft.com -td sha256 ' + `
-        '-tse "$(TrustedSigningUri)" -tsa "$(TrustedSigningAccount)" -tscp "$(TrustedSigningCertificateName)" ' + `
-        '-d "$(SigningDescription)" '
+- ${{ if and(parameters.SigningCertificate, ne(parameters.SigningCertificate, 'Unsigned')) }}:
+  - ${{ if eq(parameters.InstallTool, 'true') }}:
+    - powershell: |
+        # Install sign tool
+        dotnet tool install --global --prerelease sign
+        $signtool = (gcm sign -EA SilentlyContinue).Source
+        if (-not $signtool) {
+          $signtool = (gi "${env:USERPROFILE}\.dotnet\tools\sign.exe").FullName
+        }
+        $signargs = 'code artifact-signing -v Information ' + `
+          '-fd sha256 -t http://timestamp.acs.microsoft.com -td sha256 ' + `
+          "-ase ""${env:ASE}"" -asa ""${env:ASA}"" -ascp ""${env:ASCP}"" " + `
+          "-act azure-cli -d ""${env:DESCRIPTION}"""
+
+        Write-Host "##vso[task.setvariable variable=__TrustedSigningCmd]$signtool"
+        Write-Host "##vso[task.setvariable variable=__TrustedSigningArgs]$signargs"
+        if ($env:EXPORT_COMMAND) {
+          $signcmd = """$signtool"" $signargs"
+          Write-Host "##vso[task.setvariable variable=${env:EXPORT_COMMAND}]$signcmd"
+        }
+      workingDirectory: $(Build.BinariesDirectory)
+      displayName: 'Install Azure Artifact Signing tools'
+      env:
+        ASE: $(TrustedSigningUri)
+        ASA: $(TrustedSigningAccount)
+        ASCP: $(TrustedSigningCertificateName)
+        DESCRIPTION: $(SigningDescription)
+        EXPORT_COMMAND: ${{ parameters.ExportCommand }}
+
+  - ${{ if eq(parameters.InstallLegacyTool, 'true') }}:
+    - powershell: |
+        git clone https://github.com/python/cpython-bin-deps --revision fb06137dccc43ed5b030cdd9e3560990b37f39da --depth 1 --progress -v "signtool"
+
+        $signtool = gi signtool\x64\signtool.exe
+        $dlib = gi signtool\azure_trusted_signing\x64\Azure.CodeSigning.Dlib.dll
+        Write-Host "##vso[task.setvariable variable=MAKECAT]$(gi signtool\x64\makecat.exe)"
+
+        ConvertTo-Json @{
+            Endpoint=$env:TSE;
+            CodeSigningAccountName=$env:TSA;
+            CertificateProfileName=$env:TSCP;
+            # Only allow Azure CLI credentials
+            ExcludeCredentials=@(
+                "ManagedIdentityCredential",
+                "WorkloadIdentityCredential",
+                "SharedTokenCacheCredential",
+                "EnvironmentCredential",
+                "VisualStudioCredential",
+                "VisualStudioCodeCredential",
+                "AzurePowerShellCredential",
+                "AzureDeveloperCliCredential",
+                "InteractiveBrowserCredential"
+            );
+        } | Out-File signtool\metadata.json -Encoding ascii
+        Write-Host "##vso[task.setvariable variable=SIGNTOOL_METADATA]$(gi signtool\metadata.json)"
 
-      Write-Host "##vso[task.setvariable variable=__TrustedSigningCmd]$signtool"
-      Write-Host "##vso[task.setvariable variable=__TrustedSigningArgs]$signargs"
-      if ($env:EXPORT_COMMAND) {
-        $signcmd = """$signtool"" $signargs"
-        Write-Host "##vso[task.setvariable variable=${env:EXPORT_COMMAND}]$signcmd"
-      }
-    workingDirectory: $(Build.BinariesDirectory)
-    displayName: 'Install Trusted Signing tools'
-    env:
-      EXPORT_COMMAND: ${{ parameters.ExportCommand }}
+        $signargs = 'sign /v /fd sha256 /tr http://timestamp.acs.microsoft.com /td sha256 ' + `
+                    "/dlib ""$dlib"" /dmdf ""$(gi signtool\metadata.json)"""
+        Write-Host "##vso[task.setvariable variable=__TrustedSigningCmd]$signtool"
+        Write-Host "##vso[task.setvariable variable=__TrustedSigningArgs]$signargs"
+
+        if ($env:EXPORT_COMMAND) {
+          $signcmd = """$signtool"" $signargs"
+          Write-Host "##vso[task.setvariable variable=${env:EXPORT_COMMAND}]$signcmd"
+        }
+      workingDirectory: $(Pipeline.Workspace)
+      displayName: 'Download signtool binaries'
+      env:
+        TSE: $(TrustedSigningUri)
+        TSA: $(TrustedSigningAccount)
+        TSCP: $(TrustedSigningCertificateName)
+        EXPORT_COMMAND: ${{ parameters.ExportLegacyCommand }}
 
   - ${{ if parameters.AzureServiceConnectionName }}:
     # We sign in once with the AzureCLI task, as it uses OIDC to obtain a
@@ -45,28 +97,28 @@ steps:
         scriptType: 'ps'
         scriptLocation: 'inlineScript'
         inlineScript: |
-          "##vso[task.setvariable variable=AZURE_CLIENT_ID;issecret=true]${env:servicePrincipalId}"
-          "##vso[task.setvariable variable=AZURE_ID_TOKEN;issecret=true]${env:idToken}"
-          "##vso[task.setvariable variable=AZURE_TENANT_ID;issecret=true]${env:tenantId}"
+          "##vso[task.setvariable variable=__AZURE_CLIENT_ID;issecret=true]${env:servicePrincipalId}"
+          "##vso[task.setvariable variable=__AZURE_ID_TOKEN;issecret=true]${env:idToken}"
+          "##vso[task.setvariable variable=__AZURE_TENANT_ID;issecret=true]${env:tenantId}"
         addSpnToEnvironment: true
 
     - powershell: >
         az login --service-principal
-        -u $(AZURE_CLIENT_ID)
-        --tenant $(AZURE_TENANT_ID)
+        -u $(__AZURE_CLIENT_ID)
+        --tenant $(__AZURE_TENANT_ID)
         --allow-no-subscriptions
-        --federated-token $(AZURE_ID_TOKEN)
+        --federated-token $(__AZURE_ID_TOKEN)
       displayName: 'Authenticate signing tools (2/2)'
 
   - ${{ if parameters.Include }}:
     - powershell: |
-        if ("${{ parameters.Exclude }}") {
-          $files = (dir ${{ parameters.Include }} -Exclude ${{ parameters.Exclude }} -File)
+        if ($env:EXCLUDE) {
+          $files = (dir ($env:INCLUDE -split ';').Trim() -Exclude ($env:EXCLUDE -split ';').Trim() -File)
         } else {
-          $files = (dir ${{ parameters.Include }} -File)
+          $files = (dir ($env:INCLUDE -split ';').Trim() -File)
         }
         if ($env:FILTER) {
-          ($env:FILTER -split ';') -join "`n" | Out-File __filelist.txt -Encoding utf8
+          ($env:FILTER -split ';').Trim() -join "`n" | Out-File __filelist.txt -Encoding utf8
         } else {
           "*" | Out-File __filelist.txt -Encoding utf8
         }
@@ -82,31 +134,37 @@ steps:
         continueOnError: true
       workingDirectory: ${{ parameters.WorkingDir }}
       env:
+        INCLUDE: ${{ parameters.Include }}
+        EXCLUDE: ${{ parameters.Exclude }}
         TRUSTED_SIGNING_CMD: $(__TrustedSigningCmd)
         TRUSTED_SIGNING_ARGS: $(__TrustedSigningArgs)
         ${{ if parameters.Filter }}:
           FILTER: ${{ parameters.Filter }}
 
 
-- ${{ if parameters.ExtractDir }}:
-  - powershell: |
-      if ("${{ parameters.Exclude }}") {
-        $files = (dir ${{ parameters.Include }} -Exclude ${{ parameters.Exclude }} -File)
-      } else {
-        $files = (dir ${{ parameters.Include }} -File)
-      }
-      $c = $files | %{ (Get-AuthenticodeSignature $_).SignerCertificate } | ?{ $_ -ne $null } | select -First 1
-      if (-not $c) {
-        Write-Host "Failed to find certificate for ${{ parameters.SigningCertificate }}"
-        exit
-      }
+  - ${{ if parameters.ExtractDir }}:
+    - powershell: |
+        if ($env:EXCLUDE) {
+          $files = (dir ($env:INCLUDE -split ',').Trim() -Exclude ($env:EXCLUDE -split ',').Trim() -File)
+        } else {
+          $files = (dir ($env:INCLUDE -split ',').Trim() -File)
+        }
+        $c = $files | %{ (Get-AuthenticodeSignature $_).SignerCertificate } | ?{ $_ -ne $null } | select -First 1
+        if (-not $c) {
+          Write-Host "Failed to find certificate for ${{ parameters.SigningCertificate }}"
+          exit
+        }
 
-      $d = mkdir "${{ parameters.ExtractDir }}" -Force
-      $cf = "$d\cert.cer"
-      [IO.File]::WriteAllBytes($cf, $c.RawData)
-      $csha = (Get-FileHash $cf -Algorithm SHA256).Hash.ToLower()
+        $d = mkdir $env:EXTRACT_DIR -Force
+        $cf = "$d\cert.cer"
+        [IO.File]::WriteAllBytes($cf, $c.RawData)
+        $csha = (Get-FileHash $cf -Algorithm SHA256).Hash.ToLower()
 
-      $info = @{ Subject=$c.Subject; SHA256=$csha; }
-      $info | ConvertTo-JSON -Compress | Out-File -Encoding utf8 "$d\certinfo.json"
-    displayName: "Extract certificate info"
-    workingDirectory: ${{ parameters.WorkingDir }}
+        $info = @{ Subject=$c.Subject; SHA256=$csha; }
+        $info | ConvertTo-JSON -Compress | Out-File -Encoding utf8 "$d\certinfo.json"
+      displayName: "Extract certificate info"
+      workingDirectory: ${{ parameters.WorkingDir }}
+      env:
+        INCLUDE: ${{ parameters.Include }}
+        EXCLUDE: ${{ parameters.Exclude }}
+        EXTRACT_DIR: ${{ parameters.ExtractDir }}