Skip to content

Commit a16ccf1

Browse files
igorpecovnikigorpecovnik
committed
Add armbian-images.json generator and workflow
1 parent eb3cbd0 commit a16ccf1

File tree

2 files changed

+332
-0
lines changed

2 files changed

+332
-0
lines changed

‎.github/workflows/update-download-index.yml‎

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
name: Update download JSON index
2+
3+
on:
4+
push:
5+
repository_dispatch:
6+
types: ["Webindex update"]
7+
workflow_dispatch:
8+
9+
permissions:
10+
contents: write
11+
12+
concurrency:
13+
group: redirector
14+
cancel-in-progress: false
15+
16+
env:
17+
SOURCE_OF_TRUTH: "rsync://fi.mirror.armbian.de"
18+
19+
jobs:
20+
Webindex:
21+
name: "Generate JSON Index"
22+
runs-on: ubuntu-24.04
23+
24+
env:
25+
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
26+
27+
steps:
28+
- name: Checkout build framework repository
29+
uses: actions/checkout@v6
30+
with:
31+
repository: armbian/build
32+
fetch-depth: 1
33+
clean: false
34+
path: build
35+
36+
- name: Checkout OS repository
37+
uses: actions/checkout@v6
38+
with:
39+
repository: armbian/os
40+
fetch-depth: 1
41+
clean: false
42+
path: os
43+
44+
- name: Checkout armbian.github.io repository
45+
uses: actions/checkout@v6
46+
with:
47+
repository: armbian/armbian.github.io
48+
fetch-depth: 0
49+
clean: false
50+
path: armbian.github.io
51+
52+
- name: Checkout this repo (script)
53+
uses: actions/checkout@v6
54+
with:
55+
fetch-depth: 1
56+
clean: false
57+
path: runner
58+
59+
- name: Install dependencies
60+
run: |
61+
sudo apt-get update
62+
sudo apt-get install -y jq jc rsync
63+
64+
- name: Generate armbian-images.json
65+
working-directory: runner
66+
run: |
67+
set -euo pipefail
68+
OUT="armbian-images.json" \
69+
OS_DIR="${GITHUB_WORKSPACE}/os" \
70+
BOARD_DIR="${GITHUB_WORKSPACE}/build/config/boards" \
71+
SOURCE_OF_TRUTH="${SOURCE_OF_TRUTH}" \
72+
./scripts/generate-armbian-images-json.sh
73+
74+
# Copy into workspace root so later steps can reference easily
75+
cp armbian-images.json "${GITHUB_WORKSPACE}/armbian-images.json"
76+
77+
- name: Commit changes if any
78+
run: |
79+
80+
set -euo pipefail
81+
cd armbian.github.io
82+
83+
git config user.name "github-actions"
84+
git config user.email "github-actions@github.com"
85+
86+
git fetch origin data
87+
git checkout data
88+
git pull --rebase origin data
89+
90+
mkdir -p data
91+
cp "${GITHUB_WORKSPACE}/armbian-images.json" data/
92+
93+
git add data/armbian-images.json
94+
if ! git diff --cached --quiet; then
95+
git commit -m "Update armbian-images.json"
96+
# rebase once more right before push to reduce race window
97+
git pull --rebase origin data
98+
git push origin data
99+
fi
100+
101+
- name: Run Jira update action
102+
uses: peter-evans/repository-dispatch@v4
103+
with:
104+
token: ${{ secrets.GITHUB_TOKEN }}
105+
event-type: "Jira update"

‎scripts/generate-armbian-images-json.sh‎

Lines changed: 227 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,227 @@
1+
#!/usr/bin/env bash
2+
set -euo pipefail
3+
IFS=$'\n\t'
4+
5+
# -----------------------------------------------------------------------------
6+
# Configuration
7+
# -----------------------------------------------------------------------------
8+
SOURCE_OF_TRUTH="${SOURCE_OF_TRUTH:-rsync://fi.mirror.armbian.de}"
9+
OS_DIR="${OS_DIR:-./os}"
10+
BOARD_DIR="${BOARD_DIR:-./build/config/boards}"
11+
OUT="${OUT:-armbian-images.json}"
12+
13+
# -----------------------------------------------------------------------------
14+
# Requirements
15+
# -----------------------------------------------------------------------------
16+
need() { command -v "$1" >/dev/null || { echo "ERROR: missing '$1'" >&2; exit 1; }; }
17+
need rsync gh jq jc find grep sed cut awk sort mktemp
18+
19+
[[ -f "${OS_DIR}/exposed.map" ]] || { echo "ERROR: ${OS_DIR}/exposed.map not found" >&2; exit 1; }
20+
[[ -d "${BOARD_DIR}" ]] || { echo "ERROR: board directory not found: ${BOARD_DIR}" >&2; exit 1; }
21+
22+
# -----------------------------------------------------------------------------
23+
# Extract variable from board config
24+
# -----------------------------------------------------------------------------
25+
extract_cfg_var() {
26+
local file="$1" var="$2"
27+
awk -v var="$var" '
28+
{
29+
line=$0
30+
sub(/[ \t]*#.*/, "", line)
31+
if (line ~ var"[ \t]*=") {
32+
sub(/^.*=/,"",line)
33+
gsub(/^["'\'']|["'\'']$/,"",line)
34+
print line; exit
35+
}
36+
}' "$file" 2>/dev/null || true
37+
}
38+
39+
# -----------------------------------------------------------------------------
40+
# Load board metadata + track incomplete metadata (file-based, set -u safe)
41+
# -----------------------------------------------------------------------------
42+
declare -A BOARD_NAME_MAP
43+
declare -A BOARD_VENDOR_MAP
44+
45+
MISSING_META_FILE="$(mktemp)"
46+
trap 'rm -f "$MISSING_META_FILE"' EXIT
47+
48+
while IFS= read -r cfg; do
49+
slug="$(basename "${cfg%.*}")"
50+
slug="${slug,,}"
51+
52+
name="$(extract_cfg_var "$cfg" BOARD_NAME)"
53+
vendor="$(extract_cfg_var "$cfg" BOARD_VENDOR)"
54+
55+
[[ -n "$name" ]] && BOARD_NAME_MAP["$slug"]="$name"
56+
[[ -n "$vendor" ]] && BOARD_VENDOR_MAP["$slug"]="$vendor"
57+
58+
if [[ -z "$name" || -z "$vendor" ]]; then
59+
printf '%s\n' "$slug" >>"$MISSING_META_FILE"
60+
fi
61+
done < <(
62+
find "$BOARD_DIR" -maxdepth 1 -type f \
63+
\( -name "*.conf" -o -name "*.csc" -o -name "*.wip" -o -name "*.tvb" \) \
64+
| sort
65+
)
66+
67+
# -----------------------------------------------------------------------------
68+
# Helpers
69+
# -----------------------------------------------------------------------------
70+
is_version_token() { [[ "$1" =~ ^[0-9]{2}\.[0-9] ]]; }
71+
72+
is_preinstalled_app() {
73+
case "$1" in kali|homeassistant|openhab|omv) return 0 ;; *) return 1 ;; esac
74+
}
75+
76+
strip_img_ext() {
77+
sed -E 's/(\.img(\.(xz|zst|gz))?)$//' <<<"$1"
78+
}
79+
80+
extract_file_extension() {
81+
local n="$1"
82+
[[ "$n" == *.img.xz ]] && echo "img.xz" && return
83+
[[ "$n" == *.img.zst ]] && echo "img.zst" && return
84+
[[ "$n" == *.img.gz ]] && echo "img.gz" && return
85+
[[ "$n" == *.img ]] && echo "img" && return
86+
echo "${n##*.}"
87+
}
88+
89+
get_download_repository() {
90+
local url="$1"
91+
if [[ "$url" == https://github.com/armbian/* ]]; then
92+
awk -F/ '{print $5}' <<<"$url"
93+
elif [[ "$url" == https://dl.armbian.com/* ]]; then
94+
awk -F/ '{print $5}' <<<"$url"
95+
else
96+
echo ""
97+
fi
98+
}
99+
100+
# -----------------------------------------------------------------------------
101+
# Parse image filename
102+
# -----------------------------------------------------------------------------
103+
parse_image_name() {
104+
local name="$1"
105+
IFS="_" read -r -a p <<<"$name"
106+
107+
local ver="" board="" distro="" branch="" kernel="" tail=""
108+
local variant="server" app="" storage=""
109+
110+
if is_version_token "${p[1]:-}"; then
111+
ver="${p[1]}"; board="${p[2]}"; distro="${p[3]}"
112+
branch="${p[4]}"; kernel="${p[5]}"; tail="${p[6]:-}"
113+
else
114+
ver="${p[2]}"; board="${p[3]}"; distro="${p[4]}"
115+
branch="${p[5]}"; kernel="${p[6]}"; tail="${p[7]:-}"
116+
fi
117+
118+
if [[ "$kernel" == *-* ]]; then
119+
suffix="$(strip_img_ext "${kernel#*-}")"
120+
if is_preinstalled_app "$suffix"; then
121+
app="$suffix"
122+
else
123+
[[ "${suffix##*-}" == "ufs" ]] && storage="ufs"
124+
fi
125+
fi
126+
127+
[[ "$tail" == minimal* ]] && variant="minimal"
128+
[[ "$name" == *_desktop.img.* ]] && variant="$tail"
129+
130+
printf '%s\n' "$ver" "$board" "$distro" "$branch" "$variant" "$app" "$storage"
131+
}
132+
133+
# -----------------------------------------------------------------------------
134+
# Build feeds (NO .txt files)
135+
# -----------------------------------------------------------------------------
136+
tmpdir="$(mktemp -d)"
137+
trap 'rm -rf "$tmpdir"; rm -f "$MISSING_META_FILE"' EXIT
138+
feed="$tmpdir/feed.txt"
139+
140+
echo "▶ Building feeds…" >&2
141+
142+
# Mirror feed
143+
rsync --recursive --list-only "${SOURCE_OF_TRUTH}/dl/" |
144+
awk '
145+
{
146+
size=$2; gsub(/[.,]/,"",size)
147+
url="https://dl.armbian.com/" $5
148+
if (url ~ /\/[^\/]+\/archive\/Armbian/ &&
149+
url !~ /\.txt$/ &&
150+
url !~ /\.(asc|sha|torrent)$/ &&
151+
url !~ /(homeassistant|openhab|kali|omv)/) {
152+
dt=$3 "T" $4 "Z"; gsub("/", "-", dt)
153+
print size "|" url "|" dt
154+
}
155+
}' >"$tmpdir/a.txt"
156+
157+
# GitHub feed
158+
: >"$tmpdir/bcd.txt"
159+
for repo in community os distribution; do
160+
gh release view --json assets --repo "github.com/armbian/$repo" |
161+
jq -r '.assets[]
162+
| select(.url | test("\\.txt($|\\?)") | not)
163+
| select(.url | test("\\.(asc|sha|torrent)($|\\?)") | not)
164+
| "\(.size)|\(.url)|\(.createdAt)"' >>"$tmpdir/bcd.txt"
165+
done
166+
167+
cat "$tmpdir/a.txt" "$tmpdir/bcd.txt" >"$feed"
168+
169+
# -----------------------------------------------------------------------------
170+
# JSON generation
171+
# -----------------------------------------------------------------------------
172+
{
173+
echo '"board_slug"|"board_name"|"board_vendor"|"armbian_version"|"file_url"|"file_url_asc"|"file_url_sha"|"file_url_torrent"|"redi_url"|"redi_url_asc"|"redi_url_sha"|"redi_url_torrent"|"file_updated"|"file_size"|"distro_release"|"kernel_branch"|"image_variant"|"preinstalled_application"|"promoted"|"download_repository"|"file_extension"'
174+
175+
while IFS="|" read -r SIZE URL DATE; do
176+
IMAGE_SIZE="${SIZE//[.,]/}"
177+
IMAGE_NAME="${URL##*/}"
178+
179+
mapfile -t p < <(parse_image_name "$IMAGE_NAME")
180+
VER="${p[0]}"; BOARD="${p[1]}"; DISTRO="${p[2]}"; BRANCH="${p[3]}"
181+
VARIANT="${p[4]}"; APP="${p[5]}"; STORAGE="${p[6]}"
182+
183+
[[ -z "$BOARD" ]] && continue
184+
BOARD_SLUG="${BOARD,,}"
185+
186+
REPO="$(get_download_repository "$URL")"
187+
[[ -z "$REPO" ]] && continue
188+
189+
PREFIX=""; [[ "$REPO" == "os" ]] && PREFIX="nightly/"
190+
191+
BASE_EXT="$(extract_file_extension "$IMAGE_NAME")"
192+
if [[ "$IMAGE_NAME" == *.oowow.img.xz ]]; then
193+
FILE_EXTENSION="oowow.img.xz"
194+
elif [[ -n "$STORAGE" ]]; then
195+
FILE_EXTENSION="${STORAGE}.${BASE_EXT}"
196+
else
197+
FILE_EXTENSION="$BASE_EXT"
198+
fi
199+
200+
APP_SUFFIX=""; [[ -n "$APP" ]] && APP_SUFFIX="-${APP}"
201+
REDI_URL="https://dl.armbian.com/${PREFIX}${BOARD_SLUG}/${DISTRO^}_${BRANCH}_${VARIANT}${APP_SUFFIX}"
202+
203+
if [[ "$URL" == https://github.com/armbian/* ]]; then
204+
CACHE="https://cache.armbian.com/artifacts/${BOARD_SLUG}/archive/${IMAGE_NAME}"
205+
ASC="$CACHE.asc"; SHA="$CACHE.sha"; TOR="$CACHE.torrent"
206+
else
207+
ASC="$URL.asc"; SHA="$URL.sha"; TOR="$URL.torrent"
208+
fi
209+
210+
echo "${BOARD_SLUG}|${BOARD_NAME_MAP[$BOARD_SLUG]:-}|${BOARD_VENDOR_MAP[$BOARD_SLUG]:-}|${VER}|${URL}|${ASC}|${SHA}|${TOR}|${REDI_URL}|${REDI_URL}.asc|${REDI_URL}.sha|${REDI_URL}.torrent|${DATE}|${IMAGE_SIZE}|${DISTRO}|${BRANCH}|${VARIANT}|${APP}|false|${REPO}|${FILE_EXTENSION}"
211+
done <"$feed"
212+
213+
} | jc --csv | jq '{assets:.}' >"$OUT"
214+
215+
# -----------------------------------------------------------------------------
216+
# Emit warnings for incomplete board metadata (non-fatal)
217+
# -----------------------------------------------------------------------------
218+
if [[ -s "$MISSING_META_FILE" ]]; then
219+
echo "WARNING: Boards with incomplete metadata detected:" >&2
220+
sort -u "$MISSING_META_FILE" | while IFS= read -r slug; do
221+
[[ -z "$slug" ]] && continue
222+
echo " - ${slug} (missing BOARD_NAME and/or BOARD_VENDOR)" >&2
223+
done
224+
fi
225+
226+
echo "✔ Generated $OUT"
227+
echo "✔ Assets: $(jq '.assets | length' "$OUT")"

0 commit comments

Comments
 (0)