From 161292ff3b96f8ae00e865fd8db959f318f6b079 Mon Sep 17 00:00:00 2001 From: mohamad Date: Sun, 1 Jun 2025 16:26:49 +0200 Subject: [PATCH] refactor: Optimize Dockerfiles and deployment workflow for improved performance and reliability - Updated Dockerfiles to use `python:3.11-slim` for reduced image size and enhanced build efficiency. - Implemented multi-stage builds with selective file copying and non-root user creation for better security. - Enhanced deployment workflow with retry logic for image pushes and added cleanup steps for Docker resources. - Improved build commands with BuildKit optimizations for both backend and frontend images. --- .gitea/workflows/deploy-prod.yml | 153 +++++++++++++++++++++++++------ be/Dockerfile | 71 +++++++++----- be/Dockerfile.prod | 74 ++++++++------- 3 files changed, 213 insertions(+), 85 deletions(-) diff --git a/.gitea/workflows/deploy-prod.yml b/.gitea/workflows/deploy-prod.yml index a61bdb7..355163b 100644 --- a/.gitea/workflows/deploy-prod.yml +++ b/.gitea/workflows/deploy-prod.yml @@ -13,12 +13,10 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - - name: Install Docker - run: | - sudo apt-get update - sudo apt-get install -y docker.io + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 - name: Debug context variables run: | @@ -27,18 +25,20 @@ jobs: echo "Repository owner: ${{ gitea.repository_owner }}" echo "Event repository name: ${{ gitea.event.repository.name }}" echo "Event repository full name: ${{ gitea.event.repository.full_name }}" + echo "Event repository owner login: ${{ gitea.event.repository.owner.login }}" - - name: Build and push backend image + - name: Login to Gitea Registry env: GITEA_USERNAME: ${{ secrets.ME_USERNAME }} GITEA_PASSWORD: ${{ secrets.ME_PASSWORD }} run: | echo $GITEA_PASSWORD | docker login git.vinylnostalgia.com -u $GITEA_USERNAME --password-stdin - - # Try different context variable combinations + + - name: Set repository variables + id: vars + run: | REPO_NAME="${{ gitea.repository_name }}" ACTOR="${{ gitea.actor }}" - OWNER="${{ gitea.repository_owner }}" # Use fallback if variables are empty if [ -z "$REPO_NAME" ]; then @@ -48,42 +48,91 @@ jobs: ACTOR="${{ gitea.event.repository.owner.login }}" fi + echo "actor=$ACTOR" >> $GITHUB_OUTPUT + echo "repo_name=$REPO_NAME" >> $GITHUB_OUTPUT + echo "Using ACTOR: $ACTOR" + echo "Using REPO_NAME: $REPO_NAME" + + - name: Build backend image with optimizations + env: + GITEA_USERNAME: ${{ secrets.ME_USERNAME }} + GITEA_PASSWORD: ${{ secrets.ME_PASSWORD }} + run: | + REPO_NAME="${{ gitea.repository_name }}" + ACTOR="${{ gitea.actor }}" + + # Use fallback if variables are empty + if [ -z "$REPO_NAME" ]; then + REPO_NAME="${{ gitea.event.repository.name }}" + fi + if [ -z "$ACTOR" ]; then + ACTOR="${{ gitea.event.repository.owner.login }}" + fi + + echo "Building backend image..." echo "Using ACTOR: $ACTOR" echo "Using REPO_NAME: $REPO_NAME" - # Build with compression and smaller layers - docker build --compress --no-cache -t git.vinylnostalgia.com/$ACTOR/$REPO_NAME-backend:latest ./be -f ./be/Dockerfile.prod + # Build with BuildKit optimizations + DOCKER_BUILDKIT=1 docker build \ + --compress \ + --no-cache \ + --squash \ + --build-arg BUILDKIT_INLINE_CACHE=1 \ + -t git.vinylnostalgia.com/$ACTOR/$REPO_NAME-backend:latest \ + ./be -f ./be/Dockerfile.prod - # Push with retries - max_retries=3 + echo "Backend image built successfully" + + - name: Push backend image with retry logic + env: + GITEA_USERNAME: ${{ secrets.ME_USERNAME }} + GITEA_PASSWORD: ${{ secrets.ME_PASSWORD }} + run: | + REPO_NAME="${{ gitea.repository_name }}" + ACTOR="${{ gitea.actor }}" + + # Use fallback if variables are empty + if [ -z "$REPO_NAME" ]; then + REPO_NAME="${{ gitea.event.repository.name }}" + fi + if [ -z "$ACTOR" ]; then + ACTOR="${{ gitea.event.repository.owner.login }}" + fi + + # Push with retries and compression + max_retries=5 retry_count=0 + base_wait=10 + while [ $retry_count -lt $max_retries ]; do + echo "Pushing backend image (attempt $((retry_count + 1)) of $max_retries)..." + if docker push git.vinylnostalgia.com/$ACTOR/$REPO_NAME-backend:latest; then + echo "Backend image pushed successfully" break fi + retry_count=$((retry_count + 1)) if [ $retry_count -lt $max_retries ]; then - echo "Push failed, retrying in 10 seconds... (Attempt $retry_count of $max_retries)" - sleep 10 + wait_time=$((base_wait * retry_count)) + echo "Push failed, retrying in $wait_time seconds..." + sleep $wait_time fi done if [ $retry_count -eq $max_retries ]; then - echo "Failed to push after $max_retries attempts" + echo "Failed to push backend image after $max_retries attempts" exit 1 fi - - name: Build and push frontend image + - name: Build frontend image with optimizations env: GITEA_USERNAME: ${{ secrets.ME_USERNAME }} GITEA_PASSWORD: ${{ secrets.ME_PASSWORD }} run: | - echo $GITEA_PASSWORD | docker login git.vinylnostalgia.com -u $GITEA_USERNAME --password-stdin - - # Try different context variable combinations REPO_NAME="${{ gitea.repository_name }}" ACTOR="${{ gitea.actor }}" - OWNER="${{ gitea.repository_owner }}" # Use fallback if variables are empty if [ -z "$REPO_NAME" ]; then @@ -93,27 +142,73 @@ jobs: ACTOR="${{ gitea.event.repository.owner.login }}" fi + echo "Building frontend image..." echo "Using ACTOR: $ACTOR" echo "Using REPO_NAME: $REPO_NAME" - # Build with compression and smaller layers - docker build --compress --no-cache -t git.vinylnostalgia.com/$ACTOR/$REPO_NAME-frontend:latest ./fe -f ./fe/Dockerfile.prod + # Build with BuildKit optimizations + DOCKER_BUILDKIT=1 docker build \ + --compress \ + --no-cache \ + --squash \ + --build-arg BUILDKIT_INLINE_CACHE=1 \ + -t git.vinylnostalgia.com/$ACTOR/$REPO_NAME-frontend:latest \ + ./fe -f ./fe/Dockerfile.prod - # Push with retries - max_retries=3 + echo "Frontend image built successfully" + + - name: Push frontend image with retry logic + env: + GITEA_USERNAME: ${{ secrets.ME_USERNAME }} + GITEA_PASSWORD: ${{ secrets.ME_PASSWORD }} + run: | + REPO_NAME="${{ gitea.repository_name }}" + ACTOR="${{ gitea.actor }}" + + # Use fallback if variables are empty + if [ -z "$REPO_NAME" ]; then + REPO_NAME="${{ gitea.event.repository.name }}" + fi + if [ -z "$ACTOR" ]; then + ACTOR="${{ gitea.event.repository.owner.login }}" + fi + + # Push with retries and exponential backoff + max_retries=5 retry_count=0 + base_wait=10 + while [ $retry_count -lt $max_retries ]; do + echo "Pushing frontend image (attempt $((retry_count + 1)) of $max_retries)..." + if docker push git.vinylnostalgia.com/$ACTOR/$REPO_NAME-frontend:latest; then + echo "Frontend image pushed successfully" break fi + retry_count=$((retry_count + 1)) if [ $retry_count -lt $max_retries ]; then - echo "Push failed, retrying in 10 seconds... (Attempt $retry_count of $max_retries)" - sleep 10 + wait_time=$((base_wait * retry_count)) + echo "Push failed, retrying in $wait_time seconds..." + sleep $wait_time fi done if [ $retry_count -eq $max_retries ]; then - echo "Failed to push after $max_retries attempts" + echo "Failed to push frontend image after $max_retries attempts" exit 1 - fi \ No newline at end of file + fi + + - name: Cleanup Docker resources + if: always() + run: | + echo "Cleaning up Docker resources..." + docker system prune -af --volumes + docker logout git.vinylnostalgia.com + echo "Cleanup completed" + + - name: Show final image sizes + if: always() + run: | + echo "Final image sizes:" + docker images --format "table {{.Repository}}\t{{.Tag}}\t{{.Size}}" | grep -E "(vinylnostalgia|REPOSITORY)" \ No newline at end of file diff --git a/be/Dockerfile b/be/Dockerfile index 83238b7..53d04bd 100644 --- a/be/Dockerfile +++ b/be/Dockerfile @@ -1,7 +1,5 @@ -# be/Dockerfile - -# Use multi-stage build -FROM python:alpine AS builder +# Multi-stage build for production - optimized for size +FROM python:3.11-slim AS builder # Set environment variables ENV PYTHONDONTWRITEBYTECODE=1 \ @@ -10,34 +8,63 @@ ENV PYTHONDONTWRITEBYTECODE=1 \ PIP_DISABLE_PIP_VERSION_CHECK=1 # Install build dependencies -RUN apk add --no-cache \ +RUN apt-get update && apt-get install -y --no-install-recommends \ gcc \ - musl-dev \ - postgresql-dev + g++ \ + libpq-dev \ + && rm -rf /var/lib/apt/lists/* -# Create and activate virtual environment -RUN python -m venv /opt/venv -ENV PATH="/opt/venv/bin:$PATH" - -# Install dependencies +# Install Python dependencies COPY requirements.txt . -RUN pip install --no-cache-dir -r requirements.txt +RUN pip install --user --no-cache-dir -r requirements.txt -# Final stage -FROM python:alpine +# Production stage - minimal image +FROM python:3.11-slim AS production -# Copy virtual environment from builder -COPY --from=builder /opt/venv /opt/venv -ENV PATH="/opt/venv/bin:$PATH" +# Set environment variables +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PATH=/home/appuser/.local/bin:$PATH + +# Install only runtime dependencies +RUN apt-get update && apt-get install -y --no-install-recommends \ + libpq5 \ + curl \ + && rm -rf /var/lib/apt/lists/* \ + && apt-get clean + +# Create non-root user +RUN groupadd -g 1001 appuser && \ + useradd -u 1001 -g appuser -m appuser + +# Copy Python packages from builder stage +COPY --from=builder /root/.local /home/appuser/.local # Set working directory WORKDIR /app -# Copy application code -COPY . . +# Copy only necessary application files (be selective) +COPY --chown=appuser:appuser app/ ./app/ +COPY --chown=appuser:appuser *.py ./ +COPY --chown=appuser:appuser requirements.txt ./ + +# Create logs directory +RUN mkdir -p /app/logs && chown -R appuser:appuser /app + +# Switch to non-root user +USER appuser + +# Health check +HEALTHCHECK --interval=30s --timeout=10s --start-period=30s --retries=3 \ + CMD curl -f http://localhost:8000/health || exit 1 # Expose port EXPOSE 8000 -# Run the application -CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"] \ No newline at end of file +# Production command +CMD ["uvicorn", "app.main:app", \ + "--host", "0.0.0.0", \ + "--port", "8000", \ + "--workers", "4", \ + "--access-log", \ + "--log-level", "info"] \ No newline at end of file diff --git a/be/Dockerfile.prod b/be/Dockerfile.prod index f5915e2..53d04bd 100644 --- a/be/Dockerfile.prod +++ b/be/Dockerfile.prod @@ -1,49 +1,55 @@ -# Multi-stage build for production -FROM python:alpine AS base +# Multi-stage build for production - optimized for size +FROM python:3.11-slim AS builder # Set environment variables ENV PYTHONDONTWRITEBYTECODE=1 \ PYTHONUNBUFFERED=1 \ - PYTHONHASHSEED=random \ PIP_NO_CACHE_DIR=1 \ PIP_DISABLE_PIP_VERSION_CHECK=1 -# Install system dependencies -# Use apk for Alpine Linux instead of apt-get -RUN apk add --no-cache \ +# Install build dependencies +RUN apt-get update && apt-get install -y --no-install-recommends \ gcc \ - build-base \ - postgresql-dev \ - curl + g++ \ + libpq-dev \ + && rm -rf /var/lib/apt/lists/* -# Create non-root user (Alpine Linux style) -RUN addgroup -g 1001 -S appuser && \ - adduser -u 1001 -S appuser -G appuser - -# Development stage -FROM base AS development -WORKDIR /app +# Install Python dependencies COPY requirements.txt . -RUN pip install --no-cache-dir -r requirements.txt -COPY . . -RUN chown -R appuser:appuser /app -USER appuser -CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000", "--reload"] +RUN pip install --user --no-cache-dir -r requirements.txt -# Production stage -FROM base AS production +# Production stage - minimal image +FROM python:3.11-slim AS production + +# Set environment variables +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PATH=/home/appuser/.local/bin:$PATH + +# Install only runtime dependencies +RUN apt-get update && apt-get install -y --no-install-recommends \ + libpq5 \ + curl \ + && rm -rf /var/lib/apt/lists/* \ + && apt-get clean + +# Create non-root user +RUN groupadd -g 1001 appuser && \ + useradd -u 1001 -g appuser -m appuser + +# Copy Python packages from builder stage +COPY --from=builder /root/.local /home/appuser/.local + +# Set working directory WORKDIR /app -# Install production dependencies -COPY requirements.txt . -RUN pip install --no-cache-dir -r requirements.txt +# Copy only necessary application files (be selective) +COPY --chown=appuser:appuser app/ ./app/ +COPY --chown=appuser:appuser *.py ./ +COPY --chown=appuser:appuser requirements.txt ./ -# Copy application code -COPY . . - -# Create necessary directories and set permissions -RUN mkdir -p /app/logs && \ - chown -R appuser:appuser /app +# Create logs directory +RUN mkdir -p /app/logs && chown -R appuser:appuser /app # Switch to non-root user USER appuser @@ -55,10 +61,10 @@ HEALTHCHECK --interval=30s --timeout=10s --start-period=30s --retries=3 \ # Expose port EXPOSE 8000 -# Production command with optimizations +# Production command CMD ["uvicorn", "app.main:app", \ "--host", "0.0.0.0", \ "--port", "8000", \ - "--workers", "8", \ + "--workers", "4", \ "--access-log", \ "--log-level", "info"] \ No newline at end of file -- 2.45.2