Skip to content

Commit 198043b

Browse files
Donovan1905Donovan1905
committed
ci: generalize the build process for both lambdas
1 parent 25ee71f commit 198043b

5 files changed

Lines changed: 137 additions & 62 deletions

File tree

‎.github/workflows/package-analyzer-lambda.yml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
name: Package CUR Analyzer Lambda
2+
3+
on:
4+
push:
5+
tags:
6+
- 'analyzer-v*'
7+
8+
jobs:
9+
call-reusable-workflow:
10+
uses: ./.github/workflows/reusable-package-lambda.yml
11+
with:
12+
lambda-name: 'analyzer'
13+
lambda-path: 'lambdas/functions/iroco2-cur-analyzer'
14+
handler-dir: 'src'
15+
build-command: 'bash scripts/python-layers.sh'
16+
secrets: inherit

‎.github/workflows/package-scanner-lambda.yml‎

Lines changed: 9 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -1,54 +1,15 @@
1-
name: Package Scanner Lambda zips to GHCR
1+
name: Package Scanner Lambda
22

33
on:
44
push:
55
tags:
6-
- 'v*'
7-
8-
permissions:
9-
contents: write
6+
- 'scanner-v*'
107

118
jobs:
12-
build-and-publish:
13-
runs-on: ubuntu-latest
14-
steps:
15-
- name: Checkout
16-
uses: actions/checkout@v4
17-
18-
- name: Prepare paths
19-
id: prep
20-
run: |
21-
FUNC_DIR="lambdas/functions/iroco2-client-side-scanner"
22-
echo "FUNC_DIR=${FUNC_DIR}" >> $GITHUB_ENV
23-
echo "LAMBDA_NAME=$(basename ${FUNC_DIR})" >> $GITHUB_ENV
24-
echo "LAYERS_DIR=${FUNC_DIR}/layers" >> $GITHUB_ENV
25-
echo "HANDLER_DIR=${FUNC_DIR}/package" >> $GITHUB_ENV
26-
echo "OUT_DIR=${FUNC_DIR}/dist" >> $GITHUB_ENV
27-
mkdir -p lambdas/functions/iroco2-client-side-scanner/dist
28-
29-
- name: Create scanner-layers.zip
30-
run: |
31-
cd "$LAYERS_DIR"
32-
zip -r ../dist/scanner-layers.zip . -x "*/.*"
33-
shell: bash
34-
35-
- name: Create scanner-handler.zip
36-
run: |
37-
cd "$HANDLER_DIR"
38-
# Exclude the output zip name which Terraform previously produced to avoid recursion
39-
zip -r ../dist/scanner-handler.zip . -x "cur_scrapper.zip" "*/.*"
40-
shell: bash
41-
42-
- name: Prepare config for release
43-
run: cp ${FUNC_DIR}/lambda.yaml ${OUT_DIR}/scanner-lambda.yaml
44-
45-
- name: Create Release and Upload Assets
46-
env:
47-
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
48-
run: |
49-
gh release create ${{ github.ref_name }} \
50-
--generate-notes \
51-
--title "Release ${{ github.ref_name }}" \
52-
${OUT_DIR}/scanner-layers.zip \
53-
${OUT_DIR}/scanner-handler.zip \
54-
${OUT_DIR}/scanner-lambda.yaml
9+
call-reusable-workflow:
10+
uses: ./.github/workflows/reusable-package-lambda.yml
11+
with:
12+
lambda-name: 'scanner'
13+
lambda-path: 'lambdas/functions/iroco2-client-side-scanner'
14+
handler-dir: 'package'
15+
secrets: inherit

‎.github/workflows/reusable-package-lambda.yml‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
name: Reusable - Build and Release Lambda
2+
3+
on:
4+
workflow_call:
5+
inputs:
6+
lambda-name:
7+
description: 'The short name of the lambda (e.g., scanner, analyzer)'
8+
required: true
9+
type: string
10+
lambda-path:
11+
description: 'The path to the lambda function root directory'
12+
required: true
13+
type: string
14+
handler-dir:
15+
description: 'The name of the directory containing the handler code (e.g., package, src)'
16+
required: true
17+
type: string
18+
build-command:
19+
description: 'An optional command to run to build layers or other dependencies'
20+
required: false
21+
type: string
22+
23+
permissions:
24+
contents: write
25+
26+
jobs:
27+
build-and-release:
28+
runs-on: ubuntu-latest
29+
steps:
30+
- name: Checkout
31+
uses: actions/checkout@v4
32+
33+
- name: Set up environment
34+
run: |
35+
FUNC_DIR=${{ inputs.lambda-path }}
36+
LAYERS_DIR=${FUNC_DIR}/layers
37+
HANDLER_DIR=${FUNC_DIR}/${{ inputs.handler-dir }}
38+
OUT_DIR=${FUNC_DIR}/dist
39+
echo "FUNC_DIR=${FUNC_DIR}" >> $GITHUB_ENV
40+
echo "LAYERS_DIR=${LAYERS_DIR}" >> $GITHUB_ENV
41+
echo "HANDLER_DIR=${HANDLER_DIR}" >> $GITHUB_ENV
42+
echo "OUT_DIR=${OUT_DIR}" >> $GITHUB_ENV
43+
mkdir -p ${OUT_DIR}
44+
45+
- name: Run build script (if provided)
46+
if: "${{ inputs.build-command != '' }}"
47+
run: |
48+
cd ${{ env.FUNC_DIR }}
49+
${{ inputs.build-command }} ${{ env.OUT_DIR }}/${{ inputs.lambda-name }}-layers.zip
50+
shell: bash
51+
52+
- name: Create layers.zip
53+
if: "${{ inputs.build-command == '' }}"
54+
run: |
55+
cd "$LAYERS_DIR"
56+
zip -r ../dist/${{ inputs.lambda-name }}-layers.zip . -x "*/.*"
57+
shell: bash
58+
59+
- name: Create handler.zip
60+
run: |
61+
cd ${{ env.FUNC_DIR }}
62+
# Zip the handler directory itself for the analyzer, or the contents for others.
63+
if [[ "${{ inputs.lambda-name }}" == "analyzer" ]]; then
64+
zip -r dist/${{ inputs.lambda-name }}-handler.zip ${{ inputs.handler-dir }} -x "*/.*"
65+
else
66+
cd ${{ env.HANDLER_DIR }}
67+
zip -r ../dist/${{ inputs.lambda-name }}-handler.zip . -x "*/.*"
68+
fi
69+
shell: bash
70+
71+
- name: Prepare config for release
72+
run: cp ${{ env.FUNC_DIR }}/lambda.yaml ${{ env.OUT_DIR }}/${{ inputs.lambda-name }}-lambda.yaml
73+
74+
- name: Create Release and Upload Assets
75+
env:
76+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
77+
run: |
78+
gh release create ${{ github.ref_name }} \
79+
--generate-notes \
80+
--title "Release ${{ inputs.lambda-name }} ${{ github.ref_name }}" \
81+
${{ env.OUT_DIR }}/${{ inputs.lambda-name }}-layers.zip \
82+
${{ env.OUT_DIR }}/${{ inputs.lambda-name }}-handler.zip \
83+
${{ env.OUT_DIR }}/${{ inputs.lambda-name }}-lambda.yaml

‎lambdas/functions/iroco2-cur-analyzer/requirements.txt‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
boto3==1.35.72
2+
isodate==0.7.2
3+
pandas==2.1.4
4+
numpy==1.26.4
5+
pyarrow==14.0.1
6+
# Using compatible versions of pandas, numpy, and pyarrow to avoid dependency conflicts

‎lambdas/functions/iroco2-cur-analyzer/scripts/python-layers.sh‎

Lines changed: 23 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,24 +1,33 @@
11
#!/usr/bin/env sh
22

3-
# This script is used within GitLab CI to create the proper zip files for the layers.
4-
# This should not be ran locally when using localstack since localstack free can't use lambda layers.
3+
# This script is used within GitLab CI to create the proper zip file for the lambda layer.
54

65
set -o errexit
6+
set -o nounset
77

8-
echo "Creating zip file for lambda layers"
9-
mkdir -p ./layers
8+
if [ -z "$1" ]; then
9+
echo "Error: Output zip file path is required as the first argument." >&2
10+
exit 1
11+
fi
1012

11-
SCRIPT_PATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
13+
OUTPUT_ZIP_PATH=$1
1214

13-
# Create a single consolidated layer with all dependencies
14-
echo "Creating python directory for the layer"
15-
mkdir -p ./python
15+
echo "Creating zip file for lambda layers at ${OUTPUT_ZIP_PATH}"
1616

17-
echo "Installing all dependencies"
18-
pip install --platform manylinux2014_x86_64 --only-binary=:all: --target ./python -r requirements.txt
17+
# Create a temporary directory for building the layer to avoid polluting the source tree
18+
BUILD_DIR=$(mktemp -d)
1919

20-
echo "Compressing dependencies into a single layer"
21-
python -m zipfile -c "./layers/dependencies.zip" "./python/"
20+
# Create the python directory structure required by Lambda
21+
mkdir -p "${BUILD_DIR}/python"
2222

23-
echo "Cleaning up"
24-
rm -Rf "./python"
23+
echo "Installing all dependencies into ${BUILD_DIR}/python"
24+
pip install --platform manylinux2014_x86_64 --only-binary=:all: --target "${BUILD_DIR}/python" -r requirements.txt
25+
26+
echo "Compressing dependencies into ${OUTPUT_ZIP_PATH}"
27+
cd "${BUILD_DIR}/python"
28+
zip -r "${OUTPUT_ZIP_PATH}" .
29+
30+
# Clean up the temporary directory
31+
rm -rf "${BUILD_DIR}"
32+
33+
echo "Layer zip created successfully."

0 commit comments

Comments
 (0)