From 0d57ad7336fc5507c46e3baf326447e475eb9358 Mon Sep 17 00:00:00 2001
From: Micah Lee <micah@micahflee.com>
Date: Sat, 9 Sep 2023 09:16:56 -0700
Subject: [PATCH 01/14] Delete the old go-get folder since it no longer works
 with the modern Go ecosystem

---
 go-get/README.md                      |  85 --------------
 go-get/flatpak-go-get-generator.py    |  92 ----------------
 go-get/flatpak-go-vendor-generator.py | 152 --------------------------
 3 files changed, 329 deletions(-)
 delete mode 100644 go-get/README.md
 delete mode 100755 go-get/flatpak-go-get-generator.py
 delete mode 100755 go-get/flatpak-go-vendor-generator.py

diff --git a/go-get/README.md b/go-get/README.md
deleted file mode 100644
index ff3977ad..00000000
--- a/go-get/README.md
+++ /dev/null
@@ -1,85 +0,0 @@
-# Flatpak Go Get Generator
-Tool to automatically create the source list for a Go module.
-
-The script does not require Go in the host system.
-
-## Usage
-1. In the manifest, give the Go module network access and set GOPATH to $PWD.
-
-  Example manifest module (json):
-```json
-{
-  "name": "writeas-cli",
-  "buildsystem": "simple",
-  "build-options": {
-    "env": {
-      "GOBIN": "/app/bin/"
-    },
-    "build-args": [
-      "--share=network"
-    ]
-  },
-  "build-commands": [
-    ". /usr/lib/sdk/golang/enable.sh; export GOPATH=$PWD; go get github.com/writeas/writeas-cli/cmd/writeas"
-  ]
-}
-```
-
-  Example manifest (yaml):
-```yaml
-app-id: writeas-cli
-runtime: org.freedesktop.Platform
-runtime-version: '21.08'
-sdk: org.freedesktop.Sdk
-sdk-extensions:
-  - org.freedesktop.Sdk.Extension.golang
-command: echo "Done"
-modules:
-  - name: writeas
-    buildsystem: simple
-    build-options:
-      append-path: /usr/lib/sdk/golang/bin
-      env:
-        GOBIN: /app/bin
-        GO111MODULE: off
-        GOPATH: /run/build/writeas
-      build-args:
-        - --share=network
-    build-commands:
-      - go get github.com/writeas/writeas-cli/cmd/writeas
-```
-
-2. Run flatpak-builder with `--keep-build-dirs`.
-3. Run `go-get/flatpak-go-get-generator.py <build-dir>` with build-dir pointing the the build directory in `.flatpak-builder/build`.
-4. Convert the source list to YAML if necessary.
-5. Add the list to the sources field of the Go module in the manifest.
-6. Change build command from `go get` to `go install`.
-7. Remove network access.
-
-**The script assumes the networked built was run with `GOPATH=$PWD`.**
-
-## Example final module
-```json
-{
-      "name": "writeas-cli",
-      "buildsystem": "simple",
-      "build-options": {
-        "env": {
-          "GOBIN": "/app/bin/"
-        }
-      },
-      "build-commands": [
-        ". /usr/lib/sdk/golang/enable.sh; export GOPATH=$PWD; go install github.com/writeas/writeas-cli/cmd/writeas"
-      ],
-      "sources": [
-        {
-          "type": "git",
-          "url": "https://github.com/atotto/clipboard",
-          "commit": "aa9549103943c05f3e8951009cdb6a0bec2c8949",
-          "dest": "src/github.com/atotto/clipboard"
-        },
-        ...
-      ]
-    }
-```
-
diff --git a/go-get/flatpak-go-get-generator.py b/go-get/flatpak-go-get-generator.py
deleted file mode 100755
index 7597312a..00000000
--- a/go-get/flatpak-go-get-generator.py
+++ /dev/null
@@ -1,92 +0,0 @@
-#!/usr/bin/env python3
-# Copyright 2018 Çağatay Yiğit Şahin
-#
-# Permission is hereby granted, free of charge, to any person obtaining
-# a copy of this software and associated documentation files (the
-# "Software"), to deal in the Software without restriction, including
-# without limitation the rights to use, copy, modify, merge, publish,
-# distribute, sublicense, and/or sell copies of the Software, and to
-# permit persons to whom the Software is furnished to do so, subject to
-# the following conditions:
-#
-# The above copyright notice and this permission notice shall be included
-# in all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
-# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
-# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
-# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-
-from pathlib import Path
-from typing import List, Dict
-import subprocess
-import argparse
-import json
-
-def is_git_repository(p):
-    is_git_repo = p.is_dir() and (p / ".git").is_dir()
-    return is_git_repo
-
-def repo_paths(build_dir: Path) -> List[Path]:
-    src_dir = build_dir / 'src'
-    repo_paths: List[Path] = []
-
-    domains = src_dir.iterdir()
-    for domain in domains:
-        domain_users = domain.iterdir()
-        for user in domain_users:
-            if is_git_repository(user):
-                repo_paths.append(user)
-            else:
-                user_repos = user.iterdir()
-                for ur in user_repos:
-                    if is_git_repository(ur):
-                        repo_paths.append(ur)
-    return repo_paths
-
-def repo_source(repo_path: Path) -> Dict[str, str]:
-    def current_commit(repo_path: Path) -> str:
-        output = subprocess.check_output(['git', 'rev-parse', 'HEAD'],
-            cwd=repo_path).decode('ascii').strip()
-        return output
-
-    def remote_url(repo_path: Path) -> str:
-        output = subprocess.check_output(
-            ['git', 'remote', 'get-url', 'origin'],
-            cwd=repo_path).decode('ascii').strip()
-        return output
-    
-    repo_path_str = str(repo_path)
-    dest_path = repo_path_str[repo_path_str.rfind('src/'):]
-    source_object = {'type': 'git', 'url': remote_url(repo_path), 'commit': current_commit(repo_path), 'dest': dest_path}
-    return source_object
-
-def sources(build_dir: Path) -> List[Dict[str, str]]:
-    return list(map(repo_source, repo_paths(build_dir)))
-
-def main():
-    def directory(string: str) -> Path:
-        path = Path(string)
-        if not path.is_dir():
-            msg = 'build-dir should be a directory.'
-            raise argparse.ArgumentTypeError(msg)
-        return path
-
-    parser = argparse.ArgumentParser(description='For a Go module’s dependencies, output array of sources in flatpak-manifest format.')
-    parser.add_argument('build_dir', help='Build directory of the module in .flatpak-builder/build', type=directory)
-    parser.add_argument('-o', '--output', dest='output_file', help='The file to write the source list to. Default is <module-name>-sources.json', type=str)
-    args = parser.parse_args()
-    source_list = sources(args.build_dir)
-
-    output_file = args.output_file
-    if output_file is None:
-        output_file = args.build_dir.absolute().name + '-sources.json'
-        
-    with open(output_file, 'w') as out:
-        json.dump(source_list, out, indent=2)
-
-if __name__ == '__main__':
-    main()
diff --git a/go-get/flatpak-go-vendor-generator.py b/go-get/flatpak-go-vendor-generator.py
deleted file mode 100755
index 680a7e92..00000000
--- a/go-get/flatpak-go-vendor-generator.py
+++ /dev/null
@@ -1,152 +0,0 @@
-#!/usr/bin/env python3
-"""
-This is a very pragmatic (i.e. simple) tool for using Go vendor
-with flatpak.
-
-To make use of the tool you need to produce a vendor/modules.txt
-through `go mod vendor`.  One approach is to modify your manifest to
-include
-
-build-options:
-    build-args:
-        - --share=network
-
-and run `go mod vendor` just before you start to build.
-
-Once that is done, you should see a "vendors/modules.txt" which
-you can point this tool at.
-
-This tool has a few rough edges, such as special-casing a few things.
-For example, it assumes that everything is git-clonable.
-Except for certain URLs which are rewritten.
-
-The real solution is https://github.com/golang/go/issues/35922
-"""
-import json
-import logging
-import sys
-import urllib.request
-from html.parser import HTMLParser
-
-import attr
-
-log = logging.getLogger(__name__)
-
-@attr.s
-class GoModule:
-    name = attr.ib()
-    version = attr.ib()
-    revision = attr.ib()
-
-def parse_modules(fh):
-    for line in (l.strip() for l in fh if l.strip()):
-        log.debug("Read line: %s", line)
-        if line.startswith("# "):
-            splits = line.split(" ")
-            name, line_version = splits[-2], splits[-1]
-            if '-' in line_version:
-                log.debug("Parsing version: %s", line_version)
-                _version, date_revision = line_version.strip().split("-", 1)
-                try:
-                    log.debug("Splitting %s", date_revision)
-                    date, revision = date_revision.split('-')
-                except ValueError:
-                    log.debug("no further split of %s", date_revision)
-                    date = None
-                    version = revision = line_version
-                else:
-                    version = _version
-
-                log.debug("Parsed version into: %s %s %s", version, date, revision)
-            else:
-                revision = None
-                version = line_version
-
-            m = GoModule(name, version, revision)
-            yield m
-
-def get_go_redirect(html_data):
-    class GoImportParser(HTMLParser):
-        _repo = None
-
-        def handle_starttag(self, tag, attrs):
-            if self._repo is not None:
-                return
-
-            # Make a dict of the attribute name/values
-            # since it's easier to work with and understand.
-            _attrs = {}
-            for attr, value in attrs:
-                _attrs[attr] = value
-
-            name_attr = _attrs.get('name')
-            if name_attr != 'go-import':
-                return
-            content = _attrs.get('content')
-            if content is not None:
-                self._repo = content.split(' ')[-1]
-
-        def get_repo(self):
-            return self._repo
-
-    parser = GoImportParser()
-    parser.feed(html_data)
-    return parser.get_repo()
-
-
-def go_module_to_flatpak(m):
-    if not m.name.startswith("github.com"):
-        url = m.name
-    else:
-        splits = m.name.split('/')
-        if len(splits) > 3:
-            url = '/'.join(splits[:3])
-        else:
-            url = m.name
-    url = "https://" + url
-
-    print('Checking {}...'.format(url), file=sys.stderr)
-
-    try:
-        with urllib.request.urlopen(url + '?go-get=1') as response:
-            page_contents = str(response.read())
-    except urllib.request.URLError as e:
-        print('Failed to check {}: {}'.format(url, e), file=sys.stderr)
-        sys.exit(1)
-    else:
-        repo = get_go_redirect(page_contents)
-        url_found = repo
-        if url_found != url:
-            print(' got {}'.format(url_found), file=sys.stderr)
-        else:
-            print(' done', file=sys.stderr)
-        url = url_found
-
-    if not '+' in m.version:
-        tag = m.version
-    else:
-        splits = m.version.split('+')
-        log.debug(f"Splitting version for {url}: {m.version} {splits}")
-        tag = splits[0]
-
-    rev = m.revision
-    source = {
-        "type": "git",
-        "url": url,
-        "tag": tag,
-        "dest": "vendor/" + m.name,
-    }
-    if m.revision:
-        del source["tag"]
-        source["commit"] = m.revision
-
-    return source
-
-def main():
-    modules_file = sys.argv[1]
-    fh = open(modules_file)
-    fp_modules = [go_module_to_flatpak(m) for m in parse_modules(fh)]
-    print (json.dumps(fp_modules, indent=4))
-
-if __name__ == "__main__":
-    main()

From 7fad84cccf83cfc955ebc38202ef12b76781da73 Mon Sep 17 00:00:00 2001
From: Micah Lee <micah@micahflee.com>
Date: Sat, 9 Sep 2023 10:46:32 -0700
Subject: [PATCH 02/14] Add flatpak-go-deps.py

---
 go/README.md          | 102 ++++++++++++++++++++++++++++++++++++++++
 go/flatpak-go-deps.py | 107 ++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 209 insertions(+)
 create mode 100644 go/README.md
 create mode 100755 go/flatpak-go-deps.py

diff --git a/go/README.md b/go/README.md
new file mode 100644
index 00000000..f808b9e5
--- /dev/null
+++ b/go/README.md
@@ -0,0 +1,102 @@
+# Go
+
+This script requires Go and git to be installed.
+
+This script works by creating a new Go module in a temporary folder, add the given Go package as a dependency, and then runs `go list -m all` to get the full list of Go modules. For each module, it uses `go list -m -json <module>` to get detailed information. And then finally, it outputs the module in YAML format.
+
+## Usage
+
+```
+./flatpak-go-deps.py <repository@version>
+```
+
+For example, here's how you'd get Flatpak manifests for some Tor pluggable transports:
+
+```
+./flatpak-go-deps.py git.torproject.org/pluggable-transports/meek.git/meek-client@v0.38.0
+./flatpak-go-deps.py git.torproject.org/pluggable-transports/snowflake.git/client@v2.6.0
+./flatpak-go-deps.py gitlab.com/yawning/obfs4.git/obfs4proxy@obfs4proxy-0.0.14
+```
+
+This is what the output looks like:
+
+```
+$ ./flatpak-go-deps.py git.torproject.org/pluggable-transports/meek.git/meek-client@v0.38.0
+go: creating new go.mod: module tempmod
+Cloning into 'src/meek-client'...
+warning: redirecting to https://gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/meek.git/
+remote: Enumerating objects: 2676, done.
+remote: Counting objects: 100% (658/658), done.
+remote: Compressing objects: 100% (281/281), done.
+remote: Total 2676 (delta 372), reused 658 (delta 372), pack-reused 2018
+Receiving objects: 100% (2676/2676), 549.97 KiB | 440.00 KiB/s, done.
+Resolving deltas: 100% (1546/1546), done.
+Note: switching to 'v0.38.0'.
+
+You are in 'detached HEAD' state. You can look around, make experimental
+changes and commit them, and you can discard any commits you make in this
+state without impacting any branches by switching back to a branch.
+
+If you want to create a new branch to retain commits you create, you may
+do so (now or later) by using -c with the switch command. Example:
+
+  git switch -c <new-branch-name>
+
+Or undo this operation with:
+
+  git switch -
+
+Turn off this advice by setting config variable advice.detachedHead to false
+
+HEAD is now at 3be00b7 programVersion = "0.38.0"
+
+build-commands:
+- . /usr/lib/sdk/golang/enable.sh; export GOPATH=$PWD; export GO111MODULE=off; go
+  install git.torproject.org/pluggable-transports/meek.git/meek.git
+build-options:
+  env:
+    GOBIN: /app/bin/
+buildsystem: simple
+name: meek-client
+sources:
+- dest: src/git/torproject/org/pluggable-transports/goptlib/git
+  tag: v1.1.0
+  type: git
+  url: https://git.torproject.org/pluggable-transports/goptlib.git.git
+- dest: src/github/com/andybalholm/brotli
+  tag: v1.0.4
+  type: git
+  url: https://github.com/andybalholm/brotli.git
+- dest: src/github/com/klauspost/compress
+  tag: v1.15.9
+  type: git
+  url: https://github.com/klauspost/compress.git
+- dest: src/github/com/refraction-networking/utls
+  tag: v1.1.5
+  type: git
+  url: https://github.com/refraction-networking/utls.git
+- dest: src/golang/org/x/crypto
+  tag: v0.0.0-20220829220503-c86fa9a7ed90
+  type: git
+  url: https://golang.org/x/crypto.git
+- dest: src/golang/org/x/net
+  tag: v0.0.0-20220909164309-bea034e7d591
+  type: git
+  url: https://golang.org/x/net.git
+- dest: src/golang/org/x/sys
+  tag: v0.0.0-20220728004956-3c1f35247d10
+  type: git
+  url: https://golang.org/x/sys.git
+- dest: src/golang/org/x/term
+  tag: v0.0.0-20210927222741-03fcf44c2211
+  type: git
+  url: https://golang.org/x/term.git
+- dest: src/golang/org/x/text
+  tag: v0.3.7
+  type: git
+  url: https://golang.org/x/text.git
+- dest: src/golang/org/x/tools
+  tag: v0.0.0-20180917221912-90fa682c2a6e
+  type: git
+  url: https://golang.org/x/tools.git
+```
\ No newline at end of file
diff --git a/go/flatpak-go-deps.py b/go/flatpak-go-deps.py
new file mode 100755
index 00000000..6eb79644
--- /dev/null
+++ b/go/flatpak-go-deps.py
@@ -0,0 +1,107 @@
+#!/usr/bin/env python3
+import subprocess
+import sys
+import json
+import os
+import tempfile
+import yaml
+
+
+def get_module_info(module):
+    if " " not in module:
+        return None, None
+
+    module_name, module_version = module.split(" ", 1)
+    result = subprocess.run(
+        ["go", "list", "-m", "-json", module_name],
+        check=True,
+        capture_output=True,
+        text=True,
+    )
+    return module_name, json.loads(result.stdout)
+
+
+def main():
+    if len(sys.argv) < 2:
+        print("Usage: ./flatpak-go-deps.py <repository/folder@version>")
+        sys.exit(1)
+
+    repo_and_folder, version = sys.argv[1].split("@")
+
+    if "/" in repo_and_folder:
+        repo, folder = repo_and_folder.rsplit("/", 1)
+        repo_name = folder
+    else:
+        repo = repo_and_folder
+        folder = ""
+        repo_name = os.path.basename(repo).replace(".git", "")
+
+    with tempfile.TemporaryDirectory() as temp_dir:
+        os.chdir(temp_dir)
+
+        subprocess.run(["go", "mod", "init", "tempmod"], check=True)
+
+        try:
+            subprocess.run(
+                ["git", "clone", f"https://{repo}", f"src/{repo_name}"], check=True
+            )
+            os.chdir(f"src/{repo_name}")
+            subprocess.run(["git", "checkout", version], check=True)
+            os.chdir(temp_dir)
+            if folder:
+                os.chdir(f"src/{repo_name}/{folder}")
+        except subprocess.CalledProcessError:
+            print(
+                f"Error fetching {repo}@{version}. Please verify the repository and version."
+            )
+            sys.exit(1)
+
+        result = subprocess.run(
+            ["go", "list", "-m", "all"],
+            check=True,
+            capture_output=True,
+            text=True,
+        )
+
+        modules = result.stdout.strip().split("\n")
+        sources = []
+
+        for module in modules:
+            module_name, info = get_module_info(module)
+            if not module_name or not info:
+                continue
+            path = info["Path"]
+            version = info.get("Version")
+            if not version:
+                continue
+
+            if version.startswith("v"):
+                ref_type = "tag"
+            else:
+                ref_type = "commit"
+
+            sources.append(
+                {
+                    "type": "git",
+                    "url": f"https://{module_name}.git",
+                    ref_type: version,
+                    "dest": f"src/{path.replace('.', '/')}",
+                }
+            )
+
+        yaml_data = {
+            "name": repo_name,
+            "buildsystem": "simple",
+            "build-options": {"env": {"GOBIN": "/app/bin/"}},
+            "build-commands": [
+                f". /usr/lib/sdk/golang/enable.sh; export GOPATH=$PWD; export GO111MODULE=off; go install {repo}/{os.path.basename(repo)}"
+            ],
+            "sources": sources,
+        }
+
+        print()
+        print(yaml.dump(yaml_data))
+
+
+if __name__ == "__main__":
+    main()

From 0fcf94fe731255bf04dd05aa312333f5218a8306 Mon Sep 17 00:00:00 2001
From: Micah Lee <micah@micahflee.com>
Date: Sun, 10 Sep 2023 11:37:33 -0700
Subject: [PATCH 03/14] Update bug template to use new go script

---
 .github/ISSUE_TEMPLATE/bug_template.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE/bug_template.yml b/.github/ISSUE_TEMPLATE/bug_template.yml
index 6135b249..a3232a0c 100644
--- a/.github/ISSUE_TEMPLATE/bug_template.yml
+++ b/.github/ISSUE_TEMPLATE/bug_template.yml
@@ -36,8 +36,7 @@ body:
         - cpan/flatpak-cpan-generator.pl
         - dotnet/flatpak-dotnet-generator.py
         - dub/flatpak-dub-generator.py
-        - go-get/flatpak-go-get-generator.py
-        - go-get/flatpak-go-vendor-generator.py
+        - go/flatpak-go-deps.py
         - node/flatpak-node-generator.py
         - npm/flatpak-npm-generator.py
         - pip/flatpak-pip-generator

From f56bd4a6532af38934366335263bdef63c675094 Mon Sep 17 00:00:00 2001
From: Micah Lee <micah@micahflee.com>
Date: Sun, 10 Sep 2023 12:46:22 -0700
Subject: [PATCH 04/14] Handle detecting git URLs by making ?go-get=1 requests

---
 go/README.md          |  2 +-
 go/flatpak-go-deps.py | 32 +++++++++++++++++++++++++++++++-
 go/requirements.txt   |  3 +++
 3 files changed, 35 insertions(+), 2 deletions(-)
 create mode 100644 go/requirements.txt

diff --git a/go/README.md b/go/README.md
index f808b9e5..75ca4cb2 100644
--- a/go/README.md
+++ b/go/README.md
@@ -1,6 +1,6 @@
 # Go
 
-This script requires Go and git to be installed.
+This script requires Go and git to be installed. You also need the Python in [requirements.txt](./requirements.txt).
 
 This script works by creating a new Go module in a temporary folder, add the given Go package as a dependency, and then runs `go list -m all` to get the full list of Go modules. For each module, it uses `go list -m -json <module>` to get detailed information. And then finally, it outputs the module in YAML format.
 
diff --git a/go/flatpak-go-deps.py b/go/flatpak-go-deps.py
index 6eb79644..ff873441 100755
--- a/go/flatpak-go-deps.py
+++ b/go/flatpak-go-deps.py
@@ -5,6 +5,8 @@
 import os
 import tempfile
 import yaml
+import requests
+from bs4 import BeautifulSoup
 
 
 def get_module_info(module):
@@ -21,6 +23,30 @@ def get_module_info(module):
     return module_name, json.loads(result.stdout)
 
 
+def get_git_url(module_name):
+    if "gitlab.com" in module_name:
+        return (
+            f"https://gitlab.com/{module_name}"
+            if module_name.endswith(".git")
+            else f"https://gitlab.com/{module_name}.git"
+        )
+    elif "github.com" in module_name:
+        return (
+            f"https://github.com/{module_name}"
+            if module_name.endswith(".git")
+            else f"https://github.com/{module_name}.git"
+        )
+    else:
+        response = requests.get(f"https://{module_name}/?go-get=1")
+        if response.status_code != 200:
+            return None
+        soup = BeautifulSoup(response.content, "html.parser")
+        meta_tag = soup.find("meta", {"name": "go-import"})
+        if meta_tag:
+            return meta_tag["content"].split(" ")[2]
+        return None
+
+
 def main():
     if len(sys.argv) < 2:
         print("Usage: ./flatpak-go-deps.py <repository/folder@version>")
@@ -75,6 +101,10 @@ def main():
             if not version:
                 continue
 
+            git_url = get_git_url(module_name)
+            if not git_url:
+                git_url = f"https://{module_name}.git"
+
             if version.startswith("v"):
                 ref_type = "tag"
             else:
@@ -83,7 +113,7 @@ def main():
             sources.append(
                 {
                     "type": "git",
-                    "url": f"https://{module_name}.git",
+                    "url": git_url,
                     ref_type: version,
                     "dest": f"src/{path.replace('.', '/')}",
                 }
diff --git a/go/requirements.txt b/go/requirements.txt
new file mode 100644
index 00000000..06ec0e55
--- /dev/null
+++ b/go/requirements.txt
@@ -0,0 +1,3 @@
+pyyaml
+requests
+bs4
\ No newline at end of file

From 5384eafd14b52f47f5b6242290bdd8905fd38707 Mon Sep 17 00:00:00 2001
From: Micah Lee <micah@micahflee.com>
Date: Sun, 10 Sep 2023 13:12:25 -0700
Subject: [PATCH 05/14] Fix situation where git URLs include an extra
 github.com or gitlab.com

---
 go/README.md          | 22 +++++++++++-----------
 go/flatpak-go-deps.py |  8 ++++----
 2 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/go/README.md b/go/README.md
index 75ca4cb2..3da02a47 100644
--- a/go/README.md
+++ b/go/README.md
@@ -29,7 +29,7 @@ remote: Enumerating objects: 2676, done.
 remote: Counting objects: 100% (658/658), done.
 remote: Compressing objects: 100% (281/281), done.
 remote: Total 2676 (delta 372), reused 658 (delta 372), pack-reused 2018
-Receiving objects: 100% (2676/2676), 549.97 KiB | 440.00 KiB/s, done.
+Receiving objects: 100% (2676/2676), 549.97 KiB | 981.00 KiB/s, done.
 Resolving deltas: 100% (1546/1546), done.
 Note: switching to 'v0.38.0'.
 
@@ -62,41 +62,41 @@ sources:
 - dest: src/git/torproject/org/pluggable-transports/goptlib/git
   tag: v1.1.0
   type: git
-  url: https://git.torproject.org/pluggable-transports/goptlib.git.git
+  url: https://gitlab.torproject.org/tpo/anti-censorship.git
 - dest: src/github/com/andybalholm/brotli
   tag: v1.0.4
   type: git
-  url: https://github.com/andybalholm/brotli.git
+  url: https://github.com/github.com/andybalholm/brotli.git
 - dest: src/github/com/klauspost/compress
   tag: v1.15.9
   type: git
-  url: https://github.com/klauspost/compress.git
+  url: https://github.com/github.com/klauspost/compress.git
 - dest: src/github/com/refraction-networking/utls
   tag: v1.1.5
   type: git
-  url: https://github.com/refraction-networking/utls.git
+  url: https://github.com/github.com/refraction-networking/utls.git
 - dest: src/golang/org/x/crypto
   tag: v0.0.0-20220829220503-c86fa9a7ed90
   type: git
-  url: https://golang.org/x/crypto.git
+  url: https://go.googlesource.com/crypto
 - dest: src/golang/org/x/net
   tag: v0.0.0-20220909164309-bea034e7d591
   type: git
-  url: https://golang.org/x/net.git
+  url: https://go.googlesource.com/net
 - dest: src/golang/org/x/sys
   tag: v0.0.0-20220728004956-3c1f35247d10
   type: git
-  url: https://golang.org/x/sys.git
+  url: https://go.googlesource.com/sys
 - dest: src/golang/org/x/term
   tag: v0.0.0-20210927222741-03fcf44c2211
   type: git
-  url: https://golang.org/x/term.git
+  url: https://go.googlesource.com/term
 - dest: src/golang/org/x/text
   tag: v0.3.7
   type: git
-  url: https://golang.org/x/text.git
+  url: https://go.googlesource.com/text
 - dest: src/golang/org/x/tools
   tag: v0.0.0-20180917221912-90fa682c2a6e
   type: git
-  url: https://golang.org/x/tools.git
+  url: https://go.googlesource.com/tools
 ```
\ No newline at end of file
diff --git a/go/flatpak-go-deps.py b/go/flatpak-go-deps.py
index ff873441..3b312d65 100755
--- a/go/flatpak-go-deps.py
+++ b/go/flatpak-go-deps.py
@@ -26,15 +26,15 @@ def get_module_info(module):
 def get_git_url(module_name):
     if "gitlab.com" in module_name:
         return (
-            f"https://gitlab.com/{module_name}"
+            f"https://gitlab.com/{module_name.lstrip('gitlab.com/')}"
             if module_name.endswith(".git")
-            else f"https://gitlab.com/{module_name}.git"
+            else f"https://gitlab.com/{module_name.lstrip('gitlab.com/')}.git"
         )
     elif "github.com" in module_name:
         return (
-            f"https://github.com/{module_name}"
+            f"https://github.com/{module_name.lstrip('github.com/')}"
             if module_name.endswith(".git")
-            else f"https://github.com/{module_name}.git"
+            else f"https://github.com/{module_name.lstrip('github.com/')}.git"
         )
     else:
         response = requests.get(f"https://{module_name}/?go-get=1")

From 326c311916e823aca58521fe5ca33127804d999c Mon Sep 17 00:00:00 2001
From: Micah Lee <micah@micahflee.com>
Date: Sun, 10 Sep 2023 13:36:23 -0700
Subject: [PATCH 06/14] Handle edge-cases for git.torproject.org

---
 go/flatpak-go-deps.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/go/flatpak-go-deps.py b/go/flatpak-go-deps.py
index 3b312d65..e9a94d15 100755
--- a/go/flatpak-go-deps.py
+++ b/go/flatpak-go-deps.py
@@ -36,6 +36,8 @@ def get_git_url(module_name):
             if module_name.endswith(".git")
             else f"https://github.com/{module_name.lstrip('github.com/')}.git"
         )
+    elif "git.torproject.org" in module_name:
+        return f"https://{module_name}"
     else:
         response = requests.get(f"https://{module_name}/?go-get=1")
         if response.status_code != 200:

From 290d0636f29886a761756e2e4359aa422a3e9789 Mon Sep 17 00:00:00 2001
From: Micah Lee <micah@micahflee.com>
Date: Sun, 10 Sep 2023 14:10:49 -0700
Subject: [PATCH 07/14] Allow not specifying a version

---
 go/flatpak-go-deps.py | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/go/flatpak-go-deps.py b/go/flatpak-go-deps.py
index e9a94d15..f6e6be8f 100755
--- a/go/flatpak-go-deps.py
+++ b/go/flatpak-go-deps.py
@@ -51,10 +51,14 @@ def get_git_url(module_name):
 
 def main():
     if len(sys.argv) < 2:
-        print("Usage: ./flatpak-go-deps.py <repository/folder@version>")
+        print("Usage: ./flatpak-go-deps.py <repository/folder[@version]>")
         sys.exit(1)
 
-    repo_and_folder, version = sys.argv[1].split("@")
+    if "@" in sys.argv[1]:
+        repo_and_folder, version = sys.argv[1].split("@")
+    else:
+        repo_and_folder = sys.argv[1]
+        version = None
 
     if "/" in repo_and_folder:
         repo, folder = repo_and_folder.rsplit("/", 1)
@@ -74,14 +78,16 @@ def main():
                 ["git", "clone", f"https://{repo}", f"src/{repo_name}"], check=True
             )
             os.chdir(f"src/{repo_name}")
-            subprocess.run(["git", "checkout", version], check=True)
+
+            if version:
+                subprocess.run(["git", "checkout", version], check=True)
+
             os.chdir(temp_dir)
+
             if folder:
                 os.chdir(f"src/{repo_name}/{folder}")
         except subprocess.CalledProcessError:
-            print(
-                f"Error fetching {repo}@{version}. Please verify the repository and version."
-            )
+            print(f"Error fetching {sys.argv[1]}")
             sys.exit(1)
 
         result = subprocess.run(

From d1559d94302776d8ce2e77d6073eeda3d04abb9e Mon Sep 17 00:00:00 2001
From: Micah Lee <micah@micahflee.com>
Date: Sun, 10 Sep 2023 18:23:33 -0700
Subject: [PATCH 08/14] Support extracting the commit id from the version
 string if it is there, and otherwise clone every git repo to find its commit
 id

---
 go/flatpak-go-deps.py | 89 ++++++++++++++++++++++++++++++++++---------
 1 file changed, 71 insertions(+), 18 deletions(-)

diff --git a/go/flatpak-go-deps.py b/go/flatpak-go-deps.py
index f6e6be8f..cadd790d 100755
--- a/go/flatpak-go-deps.py
+++ b/go/flatpak-go-deps.py
@@ -3,38 +3,82 @@
 import sys
 import json
 import os
+import re
 import tempfile
 import yaml
 import requests
 from bs4 import BeautifulSoup
 
 
-def get_module_info(module):
-    if " " not in module:
-        return None, None
+def extract_commit_id(module_version):
+    # Regex to match formats like: v0.0.0-20190819201941-24fa4b261c55
+    complex_format_regex = re.compile(
+        r"v\d+\.\d+\.\d+-\d{14}-(?P<commit>[a-fA-F0-9]{12,40})"
+    )
+
+    match = complex_format_regex.search(module_version)
+    if match:
+        return match.group("commit")
+
+    # If the version is just a simple version like v1.4.0 or v0.13.0, return None
+    return None
+
+
+def get_commit_id_from_git(git_url, version=None):
+    with tempfile.TemporaryDirectory() as tmp_dir:
+        try:
+            if version:
+                print(f"Cloning {git_url}@{version}...")
+                subprocess.run(
+                    ["git", "clone", "-b", version, git_url, tmp_dir], check=True
+                )
+            else:
+                print(f"Cloning {git_url}...")
+                subprocess.run(["git", "clone", git_url, tmp_dir], check=True)
+        except subprocess.CalledProcessError:
+            # If cloning with a specific tag fails, fall back to default branch
+            if version:
+                print(f"Tag {version} not found. Cloning {git_url} default branch...")
+                subprocess.run(["git", "clone", git_url, tmp_dir], check=True)
 
-    module_name, module_version = module.split(" ", 1)
+        try:
+            result = subprocess.run(
+                ["git", "rev-parse", "HEAD"],
+                cwd=tmp_dir,
+                check=True,
+                capture_output=True,
+                text=True,
+            )
+            return result.stdout.strip()
+        except subprocess.CalledProcessError:
+            return None
+
+
+def get_module_info(module_name):
     result = subprocess.run(
         ["go", "list", "-m", "-json", module_name],
         check=True,
         capture_output=True,
         text=True,
     )
-    return module_name, json.loads(result.stdout)
+    return json.loads(result.stdout)
 
 
 def get_git_url(module_name):
+    # Remove the version suffix, if present
+    module_name = re.sub(r"/v\d+$", "", module_name)
+
     if "gitlab.com" in module_name:
         return (
-            f"https://gitlab.com/{module_name.lstrip('gitlab.com/')}"
+            f"https://gitlab.com/{module_name.replace('gitlab.com/', '')}"
             if module_name.endswith(".git")
-            else f"https://gitlab.com/{module_name.lstrip('gitlab.com/')}.git"
+            else f"https://gitlab.com/{module_name.replace('gitlab.com/', '')}.git"
         )
     elif "github.com" in module_name:
         return (
-            f"https://github.com/{module_name.lstrip('github.com/')}"
+            f"https://github.com/{module_name.replace('github.com/', '')}"
             if module_name.endswith(".git")
-            else f"https://github.com/{module_name.lstrip('github.com/')}.git"
+            else f"https://github.com/{module_name.replace('github.com/', '')}.git"
         )
     elif "git.torproject.org" in module_name:
         return f"https://{module_name}"
@@ -98,13 +142,18 @@ def main():
         )
 
         modules = result.stdout.strip().split("\n")
+        modules = modules[1:]  # Skip the first module, which is the current module
         sources = []
 
         for module in modules:
-            module_name, info = get_module_info(module)
-            if not module_name or not info:
-                continue
-            path = info["Path"]
+            module_name, module_version = module.split(" ", 1)
+            if module_version.endswith("+incompatible"):
+                module_version = module_version[:-13]
+
+            commit_id = extract_commit_id(module_version)
+
+            info = get_module_info(module_name)
+            path = info.get("Path")
             version = info.get("Version")
             if not version:
                 continue
@@ -113,16 +162,20 @@ def main():
             if not git_url:
                 git_url = f"https://{module_name}.git"
 
-            if version.startswith("v"):
-                ref_type = "tag"
-            else:
-                ref_type = "commit"
+            if not commit_id:
+                commit_id = get_commit_id_from_git(git_url, version)
+
+            if not commit_id:
+                print(
+                    f"Error: Could not retrieve commit ID for {module_name}@{version}."
+                )
+                continue
 
             sources.append(
                 {
                     "type": "git",
                     "url": git_url,
-                    ref_type: version,
+                    "commit": commit_id,
                     "dest": f"src/{path.replace('.', '/')}",
                 }
             )

From 29cc5a21fa0180e88f3af848cc0acf78b81bf812 Mon Sep 17 00:00:00 2001
From: Micah Lee <micah@micahflee.com>
Date: Sun, 10 Sep 2023 19:02:08 -0700
Subject: [PATCH 09/14] Handle subdirectories (e.g. github.com/foo/bar/subdir)

---
 go/flatpak-go-deps.py | 22 ++++++++++++----------
 1 file changed, 12 insertions(+), 10 deletions(-)

diff --git a/go/flatpak-go-deps.py b/go/flatpak-go-deps.py
index cadd790d..b7608b22 100755
--- a/go/flatpak-go-deps.py
+++ b/go/flatpak-go-deps.py
@@ -68,18 +68,16 @@ def get_git_url(module_name):
     # Remove the version suffix, if present
     module_name = re.sub(r"/v\d+$", "", module_name)
 
+    # Remove the subdirectory, if present (e.g. github.com/foo/bar/subdir -> github.com/foo/bar)
+    if "gitlab.com" in module_name or "github.com" in module_name:
+        url_parts = module_name.split("/")
+        if len(url_parts) > 3:
+            module_name = "/".join(url_parts[:3])
+
     if "gitlab.com" in module_name:
-        return (
-            f"https://gitlab.com/{module_name.replace('gitlab.com/', '')}"
-            if module_name.endswith(".git")
-            else f"https://gitlab.com/{module_name.replace('gitlab.com/', '')}.git"
-        )
+        return f"https://gitlab.com/{module_name.replace('gitlab.com/', '')}"
     elif "github.com" in module_name:
-        return (
-            f"https://github.com/{module_name.replace('github.com/', '')}"
-            if module_name.endswith(".git")
-            else f"https://github.com/{module_name.replace('github.com/', '')}.git"
-        )
+        return f"https://github.com/{module_name.replace('github.com/', '')}"
     elif "git.torproject.org" in module_name:
         return f"https://{module_name}"
     else:
@@ -162,6 +160,10 @@ def main():
             if not git_url:
                 git_url = f"https://{module_name}.git"
 
+            print(
+                f"[] module_name: {module_name}, git_url: {git_url}, version: {version}"
+            )
+
             if not commit_id:
                 commit_id = get_commit_id_from_git(git_url, version)
 

From e08a0b09fb8e4f8bef4161b227f922b28b77bab5 Mon Sep 17 00:00:00 2001
From: Micah Lee <micah@micahflee.com>
Date: Sun, 10 Sep 2023 19:27:38 -0700
Subject: [PATCH 10/14] Use Github and Gitlab APIs to save time, and also add
 lots of pretty output to facilitate debugging

---
 go/flatpak-go-deps.py | 71 +++++++++++++++++++++++++++++++++++--------
 1 file changed, 58 insertions(+), 13 deletions(-)

diff --git a/go/flatpak-go-deps.py b/go/flatpak-go-deps.py
index b7608b22..e3f78478 100755
--- a/go/flatpak-go-deps.py
+++ b/go/flatpak-go-deps.py
@@ -1,4 +1,6 @@
 #!/usr/bin/env python3
+# coding: utf-8
+
 import subprocess
 import sys
 import json
@@ -25,20 +27,50 @@ def extract_commit_id(module_version):
 
 
 def get_commit_id_from_git(git_url, version=None):
+    # If it's a GitHub URL, use the GitHub API
+    if "github.com" in git_url:
+        repo_parts = git_url.replace("https://github.com/", "").split("/")
+        if len(repo_parts) == 2:
+            owner, repo = repo_parts
+            tag_url = (
+                f"https://api.github.com/repos/{owner}/{repo}/git/refs/tags/{version}"
+            )
+            response = requests.get(tag_url)
+            if response.status_code == 200:
+                json_data = response.json()
+                commit_id = json_data["object"]["sha"]
+                print(f"✨ Used GitHub API to find commit ID: {commit_id}")
+                return commit_id
+
+    # If it's a GitLab URL, use the GitLab API
+    elif "gitlab.com" in git_url:
+        repo_parts = (
+            git_url.replace("https://gitlab.com/", "").rstrip(".git").split("/")
+        )
+        if len(repo_parts) >= 2:
+            tag_url = f"https://gitlab.com/api/v4/projects/{'%2F'.join(repo_parts)}/repository/tags/{version}"
+            response = requests.get(tag_url)
+            if response.status_code == 200:
+                json_data = response.json()
+                commit_id = json_data["commit"]["id"]
+                print(f"✨ Used GitHub API to find commit ID: {commit_id}")
+                return commit_id
+
+    # Otherwise, clone the git repo to find the commit id
     with tempfile.TemporaryDirectory() as tmp_dir:
         try:
             if version:
-                print(f"Cloning {git_url}@{version}...")
+                print(f"✨ Cloning {git_url}@{version} to find commit ID")
                 subprocess.run(
                     ["git", "clone", "-b", version, git_url, tmp_dir], check=True
                 )
             else:
-                print(f"Cloning {git_url}...")
+                print(f"✨ Cloning {git_url} to find commit ID")
                 subprocess.run(["git", "clone", git_url, tmp_dir], check=True)
         except subprocess.CalledProcessError:
             # If cloning with a specific tag fails, fall back to default branch
             if version:
-                print(f"Tag {version} not found. Cloning {git_url} default branch...")
+                print(f"✨ Tag {version} not found. Cloning {git_url} default branch...")
                 subprocess.run(["git", "clone", git_url, tmp_dir], check=True)
 
         try:
@@ -87,13 +119,19 @@ def get_git_url(module_name):
         soup = BeautifulSoup(response.content, "html.parser")
         meta_tag = soup.find("meta", {"name": "go-import"})
         if meta_tag:
-            return meta_tag["content"].split(" ")[2]
+            url = meta_tag["content"].split(" ")[2]
+            r = requests.get(url, allow_redirects=True)
+            if r.history:
+                return r.url
+            else:
+                return url
+
         return None
 
 
 def main():
     if len(sys.argv) < 2:
-        print("Usage: ./flatpak-go-deps.py <repository/folder[@version]>")
+        print("✨ Usage: ./flatpak-go-deps.py <repository/folder[@version]>")
         sys.exit(1)
 
     if "@" in sys.argv[1]:
@@ -113,15 +151,18 @@ def main():
     with tempfile.TemporaryDirectory() as temp_dir:
         os.chdir(temp_dir)
 
+        print("✨ Creating temporary Go module")
         subprocess.run(["go", "mod", "init", "tempmod"], check=True)
 
         try:
+            print("✨ Cloning the target repository")
             subprocess.run(
                 ["git", "clone", f"https://{repo}", f"src/{repo_name}"], check=True
             )
             os.chdir(f"src/{repo_name}")
 
             if version:
+                print(f"✨ Checking out version {version}")
                 subprocess.run(["git", "checkout", version], check=True)
 
             os.chdir(temp_dir)
@@ -129,7 +170,7 @@ def main():
             if folder:
                 os.chdir(f"src/{repo_name}/{folder}")
         except subprocess.CalledProcessError:
-            print(f"Error fetching {sys.argv[1]}")
+            print(f"✨ Error fetching {sys.argv[1]}")
             sys.exit(1)
 
         result = subprocess.run(
@@ -141,18 +182,24 @@ def main():
 
         modules = result.stdout.strip().split("\n")
         modules = modules[1:]  # Skip the first module, which is the current module
+
+        print(f"✨ Found {len(modules)} dependencies")
+
         sources = []
 
         for module in modules:
             module_name, module_version = module.split(" ", 1)
-            if module_version.endswith("+incompatible"):
-                module_version = module_version[:-13]
+            print(f"✨ Module: {module}")
 
             commit_id = extract_commit_id(module_version)
+            if commit_id:
+                print(f"✨ Found commit_id early: {commit_id}")
 
             info = get_module_info(module_name)
             path = info.get("Path")
             version = info.get("Version")
+            if version.endswith("+incompatible"):
+                version = version[:-13]
             if not version:
                 continue
 
@@ -160,16 +207,14 @@ def main():
             if not git_url:
                 git_url = f"https://{module_name}.git"
 
-            print(
-                f"[] module_name: {module_name}, git_url: {git_url}, version: {version}"
-            )
+            print(f"✨ Git URL: {git_url}")
 
             if not commit_id:
                 commit_id = get_commit_id_from_git(git_url, version)
 
             if not commit_id:
                 print(
-                    f"Error: Could not retrieve commit ID for {module_name}@{version}."
+                    f"✨ Error: Could not retrieve commit ID for {module_name}@{version}."
                 )
                 continue
 
@@ -192,7 +237,7 @@ def main():
             "sources": sources,
         }
 
-        print()
+        print("✨ 🌟 ✨")
         print(yaml.dump(yaml_data))
 
 

From 3d24481d480d2bae2202db3ee3fc45ab461b03ae Mon Sep 17 00:00:00 2001
From: Micah Lee <micah@micahflee.com>
Date: Sun, 10 Sep 2023 19:31:33 -0700
Subject: [PATCH 11/14] Switch to click

---
 go/README.md          |  2 +-
 go/flatpak-go-deps.py | 19 +++++++++----------
 go/requirements.txt   |  3 ++-
 3 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/go/README.md b/go/README.md
index 3da02a47..b1b36de5 100644
--- a/go/README.md
+++ b/go/README.md
@@ -1,4 +1,4 @@
-# Go
+# Flatpak Go Generator
 
 This script requires Go and git to be installed. You also need the Python in [requirements.txt](./requirements.txt).
 
diff --git a/go/flatpak-go-deps.py b/go/flatpak-go-deps.py
index e3f78478..df4670ee 100755
--- a/go/flatpak-go-deps.py
+++ b/go/flatpak-go-deps.py
@@ -7,8 +7,10 @@
 import os
 import re
 import tempfile
+
 import yaml
 import requests
+import click
 from bs4 import BeautifulSoup
 
 
@@ -129,16 +131,13 @@ def get_git_url(module_name):
         return None
 
 
-def main():
-    if len(sys.argv) < 2:
-        print("✨ Usage: ./flatpak-go-deps.py <repository/folder[@version]>")
-        sys.exit(1)
-
-    if "@" in sys.argv[1]:
-        repo_and_folder, version = sys.argv[1].split("@")
-    else:
-        repo_and_folder = sys.argv[1]
-        version = None
+@click.command()
+@click.argument("repo_and_folder", type=str)
+@click.option("--version", default=None, help="Version of the repository.")
+@click.option("--github_api_token", default=None, help="GitHub API Token.")
+@click.option("--gitlab_api_token", default=None, help="GitLab API Token.")
+def main(repo_and_folder, version, github_api_token, gitlab_api_token):
+    """Flatpak Go Generator"""
 
     if "/" in repo_and_folder:
         repo, folder = repo_and_folder.rsplit("/", 1)
diff --git a/go/requirements.txt b/go/requirements.txt
index 06ec0e55..33607f42 100644
--- a/go/requirements.txt
+++ b/go/requirements.txt
@@ -1,3 +1,4 @@
 pyyaml
 requests
-bs4
\ No newline at end of file
+bs4
+click
\ No newline at end of file

From 8a23c2eba6c8101923edfe9e5f7d5705323eef79 Mon Sep 17 00:00:00 2001
From: Micah Lee <micah@micahflee.com>
Date: Sun, 10 Sep 2023 19:48:19 -0700
Subject: [PATCH 12/14] Support github and gitlab API tokens, and make html
 parsing less brittle

---
 go/flatpak-go-deps.py | 26 +++++++++++++++++++++-----
 1 file changed, 21 insertions(+), 5 deletions(-)

diff --git a/go/flatpak-go-deps.py b/go/flatpak-go-deps.py
index df4670ee..081a640b 100755
--- a/go/flatpak-go-deps.py
+++ b/go/flatpak-go-deps.py
@@ -28,7 +28,9 @@ def extract_commit_id(module_version):
     return None
 
 
-def get_commit_id_from_git(git_url, version=None):
+def get_commit_id_from_git(
+    git_url, version=None, github_api_token=None, gitlab_api_token=None
+):
     # If it's a GitHub URL, use the GitHub API
     if "github.com" in git_url:
         repo_parts = git_url.replace("https://github.com/", "").split("/")
@@ -37,7 +39,12 @@ def get_commit_id_from_git(git_url, version=None):
             tag_url = (
                 f"https://api.github.com/repos/{owner}/{repo}/git/refs/tags/{version}"
             )
-            response = requests.get(tag_url)
+
+            headers = {}
+            if github_api_token:
+                headers["Authorization"] = f"token {github_api_token}"
+
+            response = requests.get(tag_url, headers=headers)
             if response.status_code == 200:
                 json_data = response.json()
                 commit_id = json_data["object"]["sha"]
@@ -51,6 +58,11 @@ def get_commit_id_from_git(git_url, version=None):
         )
         if len(repo_parts) >= 2:
             tag_url = f"https://gitlab.com/api/v4/projects/{'%2F'.join(repo_parts)}/repository/tags/{version}"
+
+            headers = {}
+            if gitlab_api_token:
+                headers["Private-Token"] = gitlab_api_token
+
             response = requests.get(tag_url)
             if response.status_code == 200:
                 json_data = response.json()
@@ -83,7 +95,9 @@ def get_commit_id_from_git(git_url, version=None):
                 capture_output=True,
                 text=True,
             )
-            return result.stdout.strip()
+            commit_id = result.stdout.strip()
+            print(f"✨ Found commit ID: {commit_id}")
+            return commit_id
         except subprocess.CalledProcessError:
             return None
 
@@ -121,7 +135,7 @@ def get_git_url(module_name):
         soup = BeautifulSoup(response.content, "html.parser")
         meta_tag = soup.find("meta", {"name": "go-import"})
         if meta_tag:
-            url = meta_tag["content"].split(" ")[2]
+            url = meta_tag["content"].split()[2]
             r = requests.get(url, allow_redirects=True)
             if r.history:
                 return r.url
@@ -209,7 +223,9 @@ def main(repo_and_folder, version, github_api_token, gitlab_api_token):
             print(f"✨ Git URL: {git_url}")
 
             if not commit_id:
-                commit_id = get_commit_id_from_git(git_url, version)
+                commit_id = get_commit_id_from_git(
+                    git_url, version, github_api_token, gitlab_api_token
+                )
 
             if not commit_id:
                 print(

From a9c3a3ed2637a48f29960b466f8e85773cd46c29 Mon Sep 17 00:00:00 2001
From: Micah Lee <micah@micahflee.com>
Date: Sun, 10 Sep 2023 20:21:56 -0700
Subject: [PATCH 13/14] Only use long commit IDs

---
 go/README.md          |  6 +++---
 go/flatpak-go-deps.py | 44 ++++++++++++++++++++++++++++++++-----------
 2 files changed, 36 insertions(+), 14 deletions(-)

diff --git a/go/README.md b/go/README.md
index b1b36de5..e8f5e16b 100644
--- a/go/README.md
+++ b/go/README.md
@@ -13,9 +13,9 @@ This script works by creating a new Go module in a temporary folder, add the giv
 For example, here's how you'd get Flatpak manifests for some Tor pluggable transports:
 
 ```
-./flatpak-go-deps.py git.torproject.org/pluggable-transports/meek.git/meek-client@v0.38.0
-./flatpak-go-deps.py git.torproject.org/pluggable-transports/snowflake.git/client@v2.6.0
-./flatpak-go-deps.py gitlab.com/yawning/obfs4.git/obfs4proxy@obfs4proxy-0.0.14
+./flatpak-go-deps.py git.torproject.org/pluggable-transports/meek.git/meek-client --version v0.38.0
+./flatpak-go-deps.py git.torproject.org/pluggable-transports/snowflake.git/client --version v2.6.0
+./flatpak-go-deps.py gitlab.com/yawning/obfs4.git/obfs4proxy --version obfs4proxy-0.0.14
 ```
 
 This is what the output looks like:
diff --git a/go/flatpak-go-deps.py b/go/flatpak-go-deps.py
index 081a640b..36e46b17 100755
--- a/go/flatpak-go-deps.py
+++ b/go/flatpak-go-deps.py
@@ -29,8 +29,30 @@ def extract_commit_id(module_version):
 
 
 def get_commit_id_from_git(
-    git_url, version=None, github_api_token=None, gitlab_api_token=None
+    git_url,
+    version=None,
+    short_commit_id=None,
+    github_api_token=None,
+    gitlab_api_token=None,
 ):
+    # If short_commit_id is provided, simply expand it
+    if short_commit_id:
+        print(
+            f"✨ Cloning {git_url} to find long commit ID version of {short_commit_id}"
+        )
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            subprocess.run(["git", "clone", "--bare", git_url, tmp_dir], check=True)
+            result = subprocess.run(
+                ["git", "rev-parse", short_commit_id],
+                cwd=tmp_dir,
+                check=True,
+                capture_output=True,
+                text=True,
+            )
+            commit_id = result.stdout.strip()
+            print(f"✨ Found commit ID: {commit_id}")
+            return commit_id
+
     # If it's a GitHub URL, use the GitHub API
     if "github.com" in git_url:
         repo_parts = git_url.replace("https://github.com/", "").split("/")
@@ -76,16 +98,17 @@ def get_commit_id_from_git(
             if version:
                 print(f"✨ Cloning {git_url}@{version} to find commit ID")
                 subprocess.run(
-                    ["git", "clone", "-b", version, git_url, tmp_dir], check=True
+                    ["git", "clone", "--bare", "-b", version, git_url, tmp_dir],
+                    check=True,
                 )
             else:
                 print(f"✨ Cloning {git_url} to find commit ID")
-                subprocess.run(["git", "clone", git_url, tmp_dir], check=True)
+                subprocess.run(["git", "clone", "--bare", git_url, tmp_dir], check=True)
         except subprocess.CalledProcessError:
             # If cloning with a specific tag fails, fall back to default branch
             if version:
                 print(f"✨ Tag {version} not found. Cloning {git_url} default branch...")
-                subprocess.run(["git", "clone", git_url, tmp_dir], check=True)
+                subprocess.run(["git", "clone", "--bare", git_url, tmp_dir], check=True)
 
         try:
             result = subprocess.run(
@@ -204,9 +227,9 @@ def main(repo_and_folder, version, github_api_token, gitlab_api_token):
             module_name, module_version = module.split(" ", 1)
             print(f"✨ Module: {module}")
 
-            commit_id = extract_commit_id(module_version)
-            if commit_id:
-                print(f"✨ Found commit_id early: {commit_id}")
+            short_commit_id = extract_commit_id(module_version)
+            if short_commit_id:
+                print(f"✨ Found short_commit_id: {short_commit_id}")
 
             info = get_module_info(module_name)
             path = info.get("Path")
@@ -222,10 +245,9 @@ def main(repo_and_folder, version, github_api_token, gitlab_api_token):
 
             print(f"✨ Git URL: {git_url}")
 
-            if not commit_id:
-                commit_id = get_commit_id_from_git(
-                    git_url, version, github_api_token, gitlab_api_token
-                )
+            commit_id = get_commit_id_from_git(
+                git_url, version, short_commit_id, github_api_token, gitlab_api_token
+            )
 
             if not commit_id:
                 print(

From 008a224d1c6960a7419295ac7bd38ced59254a21 Mon Sep 17 00:00:00 2001
From: Micah Lee <micah@micahflee.com>
Date: Sun, 10 Sep 2023 20:26:57 -0700
Subject: [PATCH 14/14] Update readme

---
 go/README.md | 136 +++++++++++++++++++++++++++++++++++++++++----------
 1 file changed, 109 insertions(+), 27 deletions(-)

diff --git a/go/README.md b/go/README.md
index e8f5e16b..94e58a39 100644
--- a/go/README.md
+++ b/go/README.md
@@ -18,19 +18,24 @@ For example, here's how you'd get Flatpak manifests for some Tor pluggable trans
 ./flatpak-go-deps.py gitlab.com/yawning/obfs4.git/obfs4proxy --version obfs4proxy-0.0.14
 ```
 
+If the deps are hosted on GitHub or GitLab, it uses the GitHub and GitLab API to find the commit IDs, as this is much quicker than git cloning the full repos. However these APIs have rate limits, and it falls back to git cloning. You can optionally pass in `--github_api_token` or `--gitlab_api_token` if you want to avoid the rate limits.
+
 This is what the output looks like:
 
 ```
-$ ./flatpak-go-deps.py git.torproject.org/pluggable-transports/meek.git/meek-client@v0.38.0
+$ ./flatpak-go-deps.py git.torproject.org/pluggable-transports/meek.git/meek-client --version v0.38.0
+✨ Creating temporary Go module
 go: creating new go.mod: module tempmod
+✨ Cloning the target repository
 Cloning into 'src/meek-client'...
 warning: redirecting to https://gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/meek.git/
 remote: Enumerating objects: 2676, done.
 remote: Counting objects: 100% (658/658), done.
 remote: Compressing objects: 100% (281/281), done.
 remote: Total 2676 (delta 372), reused 658 (delta 372), pack-reused 2018
-Receiving objects: 100% (2676/2676), 549.97 KiB | 981.00 KiB/s, done.
+Receiving objects: 100% (2676/2676), 549.97 KiB | 1.02 MiB/s, done.
 Resolving deltas: 100% (1546/1546), done.
+✨ Checking out version v0.38.0
 Note: switching to 'v0.38.0'.
 
 You are in 'detached HEAD' state. You can look around, make experimental
@@ -49,7 +54,84 @@ Or undo this operation with:
 Turn off this advice by setting config variable advice.detachedHead to false
 
 HEAD is now at 3be00b7 programVersion = "0.38.0"
-
+✨ Found 10 dependencies
+✨ Module: git.torproject.org/pluggable-transports/goptlib.git v1.1.0
+✨ Git URL: https://git.torproject.org/pluggable-transports/goptlib.git
+✨ Cloning https://git.torproject.org/pluggable-transports/goptlib.git@v1.1.0 to find commit ID
+Cloning into bare repository '/tmp/tmp4auq0b5l'...
+warning: redirecting to https://gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/goptlib.git/
+remote: Enumerating objects: 920, done.
+remote: Counting objects: 100% (26/26), done.
+remote: Compressing objects: 100% (20/20), done.
+remote: Total 920 (delta 8), reused 9 (delta 2), pack-reused 894
+Receiving objects: 100% (920/920), 169.14 KiB | 347.00 KiB/s, done.
+Resolving deltas: 100% (473/473), done.
+✨ Found commit ID: 781a46c66d2ddbc3509354ae7f1fccab74cb9927
+✨ Module: github.com/andybalholm/brotli v1.0.4
+✨ Git URL: https://github.com/andybalholm/brotli
+✨ Used GitHub API to find commit ID: 1d750214c25205863625bb3eb8190a51b2cef26d
+✨ Module: github.com/klauspost/compress v1.15.9
+✨ Git URL: https://github.com/klauspost/compress
+✨ Used GitHub API to find commit ID: 4b4f3c94fdf8c3a6c725e2ff110d9b44f88823ed
+✨ Module: github.com/refraction-networking/utls v1.1.5
+✨ Git URL: https://github.com/refraction-networking/utls
+✨ Used GitHub API to find commit ID: 7a37261931c6d4ab67fec65e73a3cc68df4ef84a
+✨ Module: golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90
+✨ Found short_commit_id: c86fa9a7ed90
+✨ Git URL: https://go.googlesource.com/crypto
+✨ Cloning https://go.googlesource.com/crypto to find long commit ID version of c86fa9a7ed90
+Cloning into bare repository '/tmp/tmp_v2n6aqt'...
+remote: Finding sources: 100% (6/6)
+remote: Total 6906 (delta 3826), reused 6901 (delta 3826)
+Receiving objects: 100% (6906/6906), 7.07 MiB | 12.10 MiB/s, done.
+Resolving deltas: 100% (3826/3826), done.
+✨ Found commit ID: c86fa9a7ed909e2f2a8ab8298254fca727aba16a
+✨ Module: golang.org/x/net v0.0.0-20220909164309-bea034e7d591
+✨ Found short_commit_id: bea034e7d591
+✨ Git URL: https://go.googlesource.com/net
+✨ Cloning https://go.googlesource.com/net to find long commit ID version of bea034e7d591
+Cloning into bare repository '/tmp/tmpnuomejb1'...
+remote: Finding sources: 100% (38/38)
+remote: Total 12487 (delta 7842), reused 12476 (delta 7842)
+Receiving objects: 100% (12487/12487), 13.68 MiB | 13.91 MiB/s, done.
+Resolving deltas: 100% (7842/7842), done.
+✨ Found commit ID: bea034e7d591acfddd606603cf48fae48bbdd340
+✨ Module: golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10
+✨ Found short_commit_id: 3c1f35247d10
+✨ Git URL: https://go.googlesource.com/sys
+✨ Cloning https://go.googlesource.com/sys to find long commit ID version of 3c1f35247d10
+Cloning into bare repository '/tmp/tmpq5awv07s'...
+remote: Total 14830 (delta 10172), reused 14830 (delta 10172)
+Receiving objects: 100% (14830/14830), 24.51 MiB | 18.52 MiB/s, done.
+Resolving deltas: 100% (10172/10172), done.
+✨ Found commit ID: 3c1f35247d107ad3669216fc09e75d66fa146363
+✨ Module: golang.org/x/term v0.0.0-20210927222741-03fcf44c2211
+✨ Found short_commit_id: 03fcf44c2211
+✨ Git URL: https://go.googlesource.com/term
+✨ Cloning https://go.googlesource.com/term to find long commit ID version of 03fcf44c2211
+Cloning into bare repository '/tmp/tmp047hj5wu'...
+remote: Total 385 (delta 212), reused 385 (delta 212)
+Receiving objects: 100% (385/385), 115.10 KiB | 1.55 MiB/s, done.
+Resolving deltas: 100% (212/212), done.
+✨ Found commit ID: 03fcf44c2211dcd5eb77510b5f7c1fb02d6ded50
+✨ Module: golang.org/x/text v0.3.7
+✨ Git URL: https://go.googlesource.com/text
+✨ Cloning https://go.googlesource.com/text@v0.3.7 to find commit ID
+Cloning into bare repository '/tmp/tmp83wmuo0u'...
+remote: Total 6627 (delta 3942), reused 6627 (delta 3942)
+Receiving objects: 100% (6627/6627), 24.28 MiB | 14.28 MiB/s, done.
+Resolving deltas: 100% (3942/3942), done.
+✨ Found commit ID: 383b2e75a7a4198c42f8f87833eefb772868a56f
+✨ Module: golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e
+✨ Found short_commit_id: 90fa682c2a6e
+✨ Git URL: https://go.googlesource.com/tools
+✨ Cloning https://go.googlesource.com/tools to find long commit ID version of 90fa682c2a6e
+Cloning into bare repository '/tmp/tmps_tqnqmg'...
+remote: Total 81895 (delta 50074), reused 81895 (delta 50074)
+Receiving objects: 100% (81895/81895), 51.39 MiB | 20.37 MiB/s, done.
+Resolving deltas: 100% (50074/50074), done.
+✨ Found commit ID: 90fa682c2a6e6a37b3a1364ce2fe1d5e41af9d6d
+✨ 🌟 ✨
 build-commands:
 - . /usr/lib/sdk/golang/enable.sh; export GOPATH=$PWD; export GO111MODULE=off; go
   install git.torproject.org/pluggable-transports/meek.git/meek.git
@@ -59,44 +141,44 @@ build-options:
 buildsystem: simple
 name: meek-client
 sources:
-- dest: src/git/torproject/org/pluggable-transports/goptlib/git
-  tag: v1.1.0
+- commit: 781a46c66d2ddbc3509354ae7f1fccab74cb9927
+  dest: src/git/torproject/org/pluggable-transports/goptlib/git
   type: git
-  url: https://gitlab.torproject.org/tpo/anti-censorship.git
-- dest: src/github/com/andybalholm/brotli
-  tag: v1.0.4
+  url: https://git.torproject.org/pluggable-transports/goptlib.git
+- commit: 1d750214c25205863625bb3eb8190a51b2cef26d
+  dest: src/github/com/andybalholm/brotli
   type: git
-  url: https://github.com/github.com/andybalholm/brotli.git
-- dest: src/github/com/klauspost/compress
-  tag: v1.15.9
+  url: https://github.com/andybalholm/brotli
+- commit: 4b4f3c94fdf8c3a6c725e2ff110d9b44f88823ed
+  dest: src/github/com/klauspost/compress
   type: git
-  url: https://github.com/github.com/klauspost/compress.git
-- dest: src/github/com/refraction-networking/utls
-  tag: v1.1.5
+  url: https://github.com/klauspost/compress
+- commit: 7a37261931c6d4ab67fec65e73a3cc68df4ef84a
+  dest: src/github/com/refraction-networking/utls
   type: git
-  url: https://github.com/github.com/refraction-networking/utls.git
-- dest: src/golang/org/x/crypto
-  tag: v0.0.0-20220829220503-c86fa9a7ed90
+  url: https://github.com/refraction-networking/utls
+- commit: c86fa9a7ed909e2f2a8ab8298254fca727aba16a
+  dest: src/golang/org/x/crypto
   type: git
   url: https://go.googlesource.com/crypto
-- dest: src/golang/org/x/net
-  tag: v0.0.0-20220909164309-bea034e7d591
+- commit: bea034e7d591acfddd606603cf48fae48bbdd340
+  dest: src/golang/org/x/net
   type: git
   url: https://go.googlesource.com/net
-- dest: src/golang/org/x/sys
-  tag: v0.0.0-20220728004956-3c1f35247d10
+- commit: 3c1f35247d107ad3669216fc09e75d66fa146363
+  dest: src/golang/org/x/sys
   type: git
   url: https://go.googlesource.com/sys
-- dest: src/golang/org/x/term
-  tag: v0.0.0-20210927222741-03fcf44c2211
+- commit: 03fcf44c2211dcd5eb77510b5f7c1fb02d6ded50
+  dest: src/golang/org/x/term
   type: git
   url: https://go.googlesource.com/term
-- dest: src/golang/org/x/text
-  tag: v0.3.7
+- commit: 383b2e75a7a4198c42f8f87833eefb772868a56f
+  dest: src/golang/org/x/text
   type: git
   url: https://go.googlesource.com/text
-- dest: src/golang/org/x/tools
-  tag: v0.0.0-20180917221912-90fa682c2a6e
+- commit: 90fa682c2a6e6a37b3a1364ce2fe1d5e41af9d6d
+  dest: src/golang/org/x/tools
   type: git
   url: https://go.googlesource.com/tools
 ```
\ No newline at end of file