diff --git a/.github/workflows/malicious-package-watcher.yml b/.github/workflows/malicious-package-watcher.yml new file mode 100644 index 0000000..459d73a --- /dev/null +++ b/.github/workflows/malicious-package-watcher.yml @@ -0,0 +1,86 @@ +name: Malicious Package Watcher + +on: + workflow_dispatch: + +permissions: + actions: read + contents: read + +jobs: + watch: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Determine last successful run timestamp + id: since + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + WORKFLOW_FILE: malicious-package-watcher.yml + run: | + set -euo pipefail + from=$(gh api \ + "repos/${GITHUB_REPOSITORY}/actions/workflows/${WORKFLOW_FILE}/runs?status=success&per_page=1" \ + --jq '.workflow_runs[0].run_started_at // empty') + if [[ -z "$from" ]]; then + from=$(date -u -d '1 hour ago' '+%Y-%m-%dT%H:%M:%SZ') + fi + echo "Using --from=$from" + echo "from=$from" >> "$GITHUB_OUTPUT" + + - name: Fetch malicious packages updated since last run + id: fetch + env: + OSM_QUERY: ${{ vars.OSM_QUERY }} + DOWNLOAD_THRESHOLD: ${{ vars.DOWNLOAD_THRESHOLD }} + OSM_API_KEY: ${{ secrets.OSM_API_KEY }} + run: | + set -euo pipefail + bash ./scripts/boost/malicious-package-list.sh osm \ + --query "$OSM_QUERY" \ + --threshold "$DOWNLOAD_THRESHOLD" \ + --from "${{ steps.since.outputs.from }}" \ + > packages.json + count=$(jq 'length' packages.json) + echo "Found $count package(s)" + echo "count=$count" >> "$GITHUB_OUTPUT" + + - name: Build Slack payload + if: steps.fetch.outputs.count != '0' + run: | + set -euo pipefail + jq -n \ + --argjson count ${{ steps.fetch.outputs.count }} \ + --arg from "${{ steps.since.outputs.from }}" \ + --rawfile pkgs packages.json \ + '{ + text: "New malicious package(s) detected on OpenSourceMalware", + blocks: [ + { + type: "section", + text: { + type: "mrkdwn", + text: "*\($count) new malicious package(s)* on OpenSourceMalware since `\($from)`" + } + }, + { + type: "section", + text: { + type: "mrkdwn", + text: ($pkgs[0] | map("• `\(.ecosystem):\(.name)` — \(.versions | join(", "))") | join("\n")) + } + } + ] + }' > slack-payload.json + + - name: Send Slack notification + if: steps.fetch.outputs.count != '0' + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + run: | + set -euo pipefail + curl -sSf -X POST \ + -H 'Content-Type: application/json' \ + --data @slack-payload.json \ + "$SLACK_WEBHOOK_URL" diff --git a/scripts/boost/malicious-package-list.sh b/scripts/boost/malicious-package-list.sh new file mode 100644 index 0000000..e02c399 --- /dev/null +++ b/scripts/boost/malicious-package-list.sh @@ -0,0 +1,279 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + cat < [--threshold ] [--from ] [--output ] + $0 claude --url [--model ] [--threshold ] [--output ] + $0 file --path [--threshold ] [--output ] + +Sources: + file Read the package list from a local JSON file. + --path Path to a JSON array of {name, ecosystem, versions}. + + osm Fetch the package list from the OpenSourceMalware feed. + --query JSON query body sent to the OSM search endpoint. + --from Only keep records whose updated_at is newer than this value (ISO 8601). + + claude Fetch a URL and ask the Claude API to extract vulnerable packages. + --url URL to fetch (advisory page, blog, JSON feed, etc.). + --model Claude model ID (default: claude-opus-4-7). + Requires ANTHROPIC_API_KEY in the environment. + +Common options: + --output Write the resulting JSON to instead of stdout. + --threshold Minimum download_count for a package to be included. + +EOF + exit 1 +} + +[[ $# -lt 1 ]] && usage +SOURCE="$1"; shift + +QUERY="" +DOWNLOAD_COUNT_THRESHOLD="" +PATH_INPUT="" +URL_INPUT="" +MODEL="claude-opus-4-7" +OUTPUT_FILE="" +FROM_INPUT="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --query) QUERY="$2"; shift 2 ;; + --threshold) DOWNLOAD_COUNT_THRESHOLD="$2"; shift 2 ;; + --url) URL_INPUT="$2"; shift 2 ;; + --model) MODEL="$2"; shift 2 ;; + --path) PATH_INPUT="$2"; shift 2 ;; + --output) OUTPUT_FILE="$2"; shift 2 ;; + --from) FROM_INPUT="$2"; shift 2 ;; + -h|--help) usage ;; + *) echo "Unknown argument: $1" >&2; usage ;; + esac +done + +case "$SOURCE" in + file) + [[ -z "$PATH_INPUT" ]] && usage + [[ ! -f "$PATH_INPUT" ]] && { echo "File not found: $PATH_INPUT" >&2; exit 1; } + + PACKAGE_VERSIONS=$(jq '.' "$PATH_INPUT") + ;; + osm) + [[ -z "$QUERY" || -z "$DOWNLOAD_COUNT_THRESHOLD" || -z "$OSM_API_KEY" ]] && usage + + QUERY=$(echo "$QUERY" | jq '.page = (.page//1) | .limit = (.limit//100)') + + PACKAGE_VERSIONS="[]" + PAGE=1 + TOTAL_PAGES=1 + +read -r -d '' JQ_CMD << 'EOM' || true + [(.data//[])[] | + select(.status == "verified" and ($from == "" or (.verified_at[:19] + "Z" | fromdateiso8601) > ($from | fromdateiso8601)))] | + group_by(.resource_identifier) | + [ + .[] | + { + "name" : .[0].package_name, + "ecosystem": .[0].registry, + "versions": [.[].version_info], + "download_count_total": .[0].download_count + } + ] +EOM + + while :; do + PAGED_QUERY=$(echo "$QUERY" | jq --argjson p "$PAGE" '.page = $p') + + RESPONSE=$(curl -s 'https://zyqmpfcrijqmwyzbkubf.supabase.co/functions/v1/query-search' \ + -H 'apikey: $OSM_API_KEY' \ + -H 'authorization: Bearer $OSM_API_KEY' \ + -H 'content-type: application/json' \ + -H 'origin: https://opensourcemalware.com' \ + -H 'referer: https://opensourcemalware.com/' \ + -H 'x-client-info: supabase-js-web/2.56.0' \ + --data-raw "$PAGED_QUERY") + + PAGE_DATA=$(echo "$RESPONSE" | jq --arg from "$FROM_INPUT" "$JQ_CMD") + PACKAGE_VERSIONS=$(echo "[$PACKAGE_VERSIONS,$PAGE_DATA]" | jq '.[0] + .[1]') + + TOTAL_PAGES=$(echo "$RESPONSE" | jq -r '.totalPages // 1') + [[ "$PAGE" -ge "$TOTAL_PAGES" ]] && break + PAGE=$((PAGE + 1)) + done + + ;; + + file) + [[ -z "$PATH_INPUT" ]] && usage + [[ ! -f "$PATH_INPUT" ]] && { echo "File not found: $PATH_INPUT" >&2; exit 1; } + + PACKAGE_VERSIONS=$(jq '.' "$PATH_INPUT") + ;; + + claude) + [[ -z "$URL_INPUT" ]] && usage + [[ -z "${ANTHROPIC_API_KEY:-}" ]] && { echo "ANTHROPIC_API_KEY env var is required" >&2; exit 1; } + + tmp_output=$(mktemp boost.XXXXXX) + URL_CONTENT=$(curl -fsSL -o "$tmp_output" --max-time 60 "$URL_INPUT") \ + || { echo "Failed to fetch URL: $URL_INPUT" >&2; exit 1; } + + JQ_CMD='{ + model: $model, + max_tokens: 16000, + system: "You are a security analyst. From the provided content, extract every vulnerable, malicious, or compromised software package referenced. Return strictly the structured JSON defined by the schema. The ecosystem must be the package registry name in lowercase (npm, pypi, maven, rubygems, nuget, go, packagist, cargo, etc.). If no vulnerable packages are present, return an empty array.", + output_config: { + format: { + type: "json_schema", + schema: { + type: "object", + properties: { + packages: { + type: "array", + items: { + type: "object", + properties: { + name: {type: "string"}, + ecosystem: {type: "string"}, + versions: {type: "array", items: {type: "string"}} + }, + required: ["name", "ecosystem", "versions"], + additionalProperties: false + } + } + }, + required: ["packages"], + additionalProperties: false + } + } + }, + messages: [{ + role: "user", + content: "Source URL: \($url)\n\nContent:\n\($content)" + }] + }' + + REQUEST_BODY_FILE=$(mktemp boost-request.XXXXXX) + echo "{}" | jq \ + --arg model "$MODEL" \ + --arg url "$URL_INPUT" \ + --rawfile content "$tmp_output" \ + "$JQ_CMD" \ + > "$REQUEST_BODY_FILE" + + RESPONSE=$(curl -sS https://api.anthropic.com/v1/messages \ + -H "Content-Type: application/json" \ + -H "x-api-key: $ANTHROPIC_API_KEY" \ + -H "anthropic-version: 2023-06-01" \ + -d "@$REQUEST_BODY_FILE") + + if echo "$RESPONSE" | jq -e '.type == "error"' > /dev/null 2>&1; then + echo "Claude API error:" >&2 + echo "$RESPONSE" | jq '.error' >&2 + exit 1 + fi + + EXTRACTED=$(echo "$RESPONSE" | jq -r 'first(.content[] | select(.type == "text") | .text)') + [[ -z "$EXTRACTED" || "$EXTRACTED" == "null" ]] && { echo "No text content in Claude response" >&2; echo "$RESPONSE" >&2; exit 1; } + + PACKAGE_VERSIONS=$(echo "$EXTRACTED" | jq '.packages') + ;; + stdin) + + [[ -t 0 ]] && { echo "stdin source requires JSON input piped to standard input" >&2; exit 1; } + PACKAGE_VERSIONS=$(jq '.') \ + || { echo "Failed to parse JSON from stdin" >&2; exit 1; } + ;; + + *) + usage + ;; +esac + +PKG_COUNT=$(echo "$PACKAGE_VERSIONS" | jq 'length') + +# Enrich npm packages missing download_count_total / download_count_latest +# using the npmjs last-week per-version downloads endpoint. +for i in $(seq 0 $((PKG_COUNT - 1))); do + [[ $PKG_COUNT -eq 0 ]] && break + PKG=$(echo "$PACKAGE_VERSIONS" | jq ".[$i]") + + HAS_TOTAL=$(echo "$PKG" | jq 'has("download_count_total")') + HAS_LATEST=$(echo "$PKG" | jq 'has("download_count_latest")') + + if [[ "$HAS_TOTAL" == "true" || "$HAS_LATEST" == "true" ]]; then + continue + fi + + ECOSYSTEM=$(echo "$PKG" | jq -r '.ecosystem') + [[ "$ECOSYSTEM" != "npm" ]] && continue + + NAME=$(echo "$PKG" | jq -r '.name') + ENCODED_NAME=$(jq -rn --arg n "$NAME" '$n | @uri') + + NPM_RESPONSE=$(curl -sS "https://api.npmjs.org/versions/$ENCODED_NAME/last-week") || continue + echo "$NPM_RESPONSE" | jq -e '.downloads' > /dev/null 2>&1 || continue + + TOTAL=$(echo "$NPM_RESPONSE" | jq '[.downloads | to_entries[].value] | add // 0') + LATEST=$(echo "$NPM_RESPONSE" | jq ' + .downloads | to_entries + | if length == 0 then 0 + else (max_by(.key | split(".") | map(tonumber? // 0)) | .value) + end + ') + + PKG=$(echo "$PKG" | jq \ + --argjson total "$TOTAL" \ + --argjson latest "$LATEST" \ + '. + {download_count_total: $total, download_count_latest: $latest}') + PACKAGE_VERSIONS=$(echo "$PACKAGE_VERSIONS" | jq --argjson id "$i" --argjson p "$PKG" '.[$id] = $p') +done + +if [[ -n "$DOWNLOAD_COUNT_THRESHOLD" ]]; then + PACKAGE_VERSIONS=$(echo "$PACKAGE_VERSIONS" | jq --argjson count "$DOWNLOAD_COUNT_THRESHOLD" -r '[.[] | select(.download_count_total >= $count)]') +fi + +echo "$PACKAGE_VERSIONS" + +exit 0 + +# Check each package against the OSV.dev database (POST /v1/query). +for i in $(seq 0 $((PKG_COUNT - 1))); do + [[ $PKG_COUNT -eq 0 ]] && break + PKG=$(echo "$PACKAGE_VERSIONS" | jq ".[$i]") + NAME=$(echo "$PKG" | jq -r '.name') + ECOSYSTEM=$(echo "$PKG" | jq -r '.ecosystem') + + FOUND_VULN=false + VULNC_COUNT=$(echo "$PKG" | jq -r '.vulns | length') + if [[ "$VULNC_COUNT" -gt 0 ]]; then + continue + fi + + while IFS= read -r VERSION; do + [[ -z "$VERSION" ]] && continue + OSV_BODY=$(jq -n --arg n "$NAME" --arg e "$ECOSYSTEM" --arg v "$VERSION" \ + '{package: {name: $n, ecosystem: $e}, version: $v}') + OSV_RESPONSE=$(curl -sS -X POST https://api.osv.dev/v1/query \ + -H 'Content-Type: application/json' \ + -d "$OSV_BODY") || continue + + VULN_IDS=$(echo "$OSV_RESPONSE" | jq '[(.vulns // [])[] | .id]') + if [[ "$VULN_IDS" != "[]" ]]; then + PKG=$(echo "$PKG" | jq --argjson vuls "$VULN_IDS" '. + {vulns: $vuls}') + PACKAGE_VERSIONS=$(echo "$PACKAGE_VERSIONS" | jq --argjson id "$i" --argjson p "$PKG" '.[$id] = $p') + break + fi + done < <(echo "$PKG" | jq -r '.versions[]') +done + + +if [[ -n "$OUTPUT_FILE" ]]; then + printf '%s\n' "$PACKAGE_VERSIONS" > "$OUTPUT_FILE" +else + echo "$PACKAGE_VERSIONS" +fi