Initial commit for v1

README.md

@@ -1,12 +1,10 @@
 # ArchRepo
-## Version 2
+## Version 1
-
-### Past versions
-* Version 1 is the current version, running state-fully in a dedicated VM, which uses NFS to mount the repository (and
-thus works only on a local network)
 
 ### Introduction
-Version 2 targets an all-new architecture:
+In a nutshell:
-* written in Python
+* this script is run punctually, on a dedicated Arch VM;
-* running in a container
+* Proces:
-  * potentially a stateless and ephemeral container
+  * it takes all packages in the repository;
+  * checks online if a new version is available
+  * if so build it and then add it to the repository

parser.awk

@@ -0,0 +1,17 @@
+BEGIN {
+  printit = 0
+}
+
+{
+  if (printit == 1) {
+    print
+    printit = 0
+  }
+  if (/%FILENAME%/) {
+    printf "|"
+    printit = 1
+  }
+  if (/%NAME%/ || /%VERSION%/) {
+    printit = 1
+  }
+}

update-repo.py

@@ -1,80 +0,0 @@
-# KTO ArchRepo update script
-# version 2.0
-
-# The purpose of this script is to automate the maintenance of a custom pacman (Arch Linux) repository, whose goal is to
-# provide already built packages of AUR software.
-
-# This version 2 aims at 2 major evolutions:
-# * manage dependencies (hence Python and object-oriented programming);
-# * run this script in a stateless Arch container, rather than a full VM as in version 1.
-# Additionaly, we may like to manage working directory in a better way (check available storage space, and existing
-# directories).
-
-# This script should NOT be run as root. Or should it?
-# * We are in a stateless container, but I don't think we can run makepkg as root...
-
-# Imports
-import os
-
-# Class for AUR package
-class Package:
-  def __init__(self, name):
-    self.name = name
-    # + version, dependencies (which should contain version as well)
-
-def parseDb(db_path):
-  # here we should use AWK to parse our repo database file and populate the collection a collection of packages to build
-  # that we then return
-
-def checkUpdates(packages_to_build): # or checkDependencies?
-  # here we should check for all packages in packages_to_build if they need an update and:
-  # * remove those which do not from the collection
-  # * check (recursively) for all these packages_to_build if they have dependencies
-  #   * if AUR dependencies, add them to this collection
-  #   * if Arch dependencies, add them to another collection arch_packages_to_install
-
-def installArchPackages(packages):
-  # packages is a collection of packages (objects or names?) to install
-  os.system("sudo pacman -Syu " + packages.toSpaceString()) # [].toSpaceString() being to define...
-
-def buildAurPackage(package):
-  # Fetch package (git clone)
-  # cd to directory
-  os.system("makepkg -s --no-confirm --no-progress-bar")
-
-def addPackageToRepo(package, repo):
-  os.system("cp...")
-  os.system("repo-add...")
-
-def sortByDependencies():
-  # packages_to_build should be sorted by dependencies here
-  # packages that are dependencies should be tagged adequately
-
-def init():
-  # Check for main dependencies:
-  deps = ['git', 'awk', 'package-query']
-  installArchPackages(deps)
-
-# This function is a "main" function
-def updateRepo():
-  init()
-  packages_to_build = parseDb(db_path)
-  checkUpdates() # ideally, this should directly edit packages_to_build AND arch_packages_to_install (I believe we can return tuples in Python)
-  installArchPackages(arch_packages_to_install)
-  sortByDependencies()
-  for package in packages_to_build:
-    buildAurPackage(package)
-    addPackageToRepo(package, repo)
-    if package.isDependency:
-      installArchPackages([package])
-
-def newPackage(package):
-  init()
-  checkUpdates([package])
-  installArchPackages(arch_packages_to_install)
-  sortByDependencies()
-  for package in packages_to_build:
-    buildAurPackage(package)
-    addPackageToRepo(package, repo)
-    if package.isDependency:
-      installArchPackages([package])

update-repo.sh

@@ -0,0 +1,263 @@
+#!/bin/bash
+
+# The purpose of this script is to automate the maintenance of a custom pacman (Arch Linux) repository, whose goal is to
+# provide package-ready AUR softs.
+# This script should NOT be run as root.
+
+# Dependencies
+# Make sure that AWK rules file is present, readable and correctly set in this script's parameters
+# Make sure the following packages are installed before running this script:
+# package-query aurutils git
+
+# Functions
+
+##################################################################
+ascii_frag() {
+    expr match "$1" "\([^[:digit:]]*\)"
+}
+
+ascii_remainder() {
+    expr match "$1" "[^[:digit:]]*\(.*\)"
+}
+
+numeric_frag() {
+    expr match "$1" "\([[:digit:]]*\)"
+}
+
+numeric_remainder() {
+    expr match "$1" "[[:digit:]]*\(.*\)"
+}
+
+# return 1 for $1 > $2
+# return 2 for $1 < $2
+# return 0 for equal
+vercomp() {
+    local WORK1="$1"
+    local WORK2="$2"
+    local NUM1="", NUM2="", ASCII1="", ASCII2=""
+    while true; do
+        ASCII1=`ascii_frag "${WORK1}"`
+        ASCII2=`ascii_frag "${WORK2}"`
+        WORK1=`ascii_remainder "${WORK1}"`
+        WORK2=`ascii_remainder "${WORK2}"`
+
+        if [ "${ASCII1}" \> "${ASCII2}" ]; then
+            return 1
+        elif [ "${ASCII1}" \< "${ASCII2}" ]; then
+            return 2
+        fi
+
+        NUM1=`numeric_frag "${WORK1}"`
+        NUM2=`numeric_frag "${WORK2}"`
+        WORK1=`numeric_remainder "${WORK1}"`
+        WORK2=`numeric_remainder "${WORK2}"`
+
+        if [ -z "${NUM1}" -a -z "${NUM2}" ]; then
+            return 0
+        elif [ -z "${NUM1}" -a -n "${NUM2}" ]; then
+            return 2
+        elif [ -n "${NUM1}" -a -z "${NUM2}" ]; then
+            return 1
+        fi
+
+        if [ "${NUM1}" -gt "${NUM2}" ]; then
+            return 1
+        elif [ "${NUM1}" -lt "${NUM2}" ]; then
+            return 2
+        fi
+    done
+}
+##################################################################
+
+init() {
+  # Only run if the user is not root
+  if [[ $USER = 'root' ]] ; then
+    echo "You cannot run this script as root!"
+    exit 1
+  fi
+
+  if [ -n "$(ls $run_directory_path)" ] ; then
+    echo "The running directory is not empty!"
+    exit 1
+  fi
+
+  # TODO Check if required packages are installed
+
+  echo "ArchRepo update script"
+}
+
+checkUpdates() {
+  echo "Checking for updates:"
+  upgraded_packages=""
+  stream=$(tar xOf $repo_directory/$repo_db_file --wildcards */desc | awk -f $awk_rules_file)
+
+  OIFS=$IFS
+  IFS='|'
+  stream="${stream:1}"
+  for package in $stream
+  do
+    IFS='
+'
+    count=0
+    for line in $package; do
+      if [ $count -eq 0 ]; then
+        filename=$line
+      elif [ $count -eq 1 ]; then
+        name=$line
+      elif [ $count -eq 2 ]; then
+        version=$line
+      fi
+
+      ((count++))
+    done
+
+    reg="(linux-lts).+-((docs)|(headers))"
+    if [[ $name =~ $reg ]]; then
+      echo -e "\nSkipping $name ($version, default exception)."
+      continue 1
+    fi
+
+    exlist=""
+    if echo $exlist | grep -w $name > /dev/null; then
+      echo -e "\nSkipping $name ($version, temporary exception)."
+      continue 1
+    fi
+
+    echo -e "\nChecking $name ($version)..."
+    aur_version=$(package-query -A -f %v $name)
+
+    vercomp $aur_version $version
+    result=$?
+    if [[ $result -eq 1 ]]; then
+      echo "New version for $name: $version -> $aur_version"
+      createPackage $name
+      addPackageToRepo $name $aur_version
+      upgraded_packages="$upgraded_packages $name"
+    fi
+
+    IFS='|'
+  done
+  IFS=$OIFS
+  echo "All packages processed."
+
+  if [[ $upgraded_packages != "" ]]; then
+    echo "The following packages were upgraded:"
+    echo $upgraded_packages
+  fi
+}
+
+addPackage() {
+  createPackage $1
+  addPackageToRepo $1
+}
+
+createPackage() { # Create package $1
+  echo "Fetching $1 from AUR..."
+  aur fetch $1 # TODO This could be replaced by a git clone or even a wget+tar thanks to package-query
+  echo "Making $1 package..."
+  cd $1
+  # if linux-lts{414,49} then download kernel source from local repo (saves ~100MB each time)
+  # WARNING this onlys works because we already have said sources AND because PKBUILD follow this particular line
+  # (not the same in 419 nor 54)
+  reg="(linux-lts).+"
+  if [[ $1 =~ $reg ]]; then
+    sed -i -e "s/\"https:\/\/www.kernel.org\/pub\/linux\/kernel\/v4.x\/\${_srcname}.tar.xz\"/\"https:\/\/archlinux.kto.black\/resources\/\${_srcname}.tar.xz\"/g" ./PKGBUILD
+  fi
+  makepkg -s --noconfirm --noprogressbar
+  cd $run_directory_path
+}
+
+addPackageToRepo() { # Add package $1
+  echo "Adding $1 to repository..."
+  cp $run_directory_path/$1/$1-$2*.pkg.tar.xz $repo_directory
+  repo-add $repo_directory/$repo_db_file $repo_directory/$1-$2*.pkg.tar.xz
+  reg="(linux-lts).+"
+  if [[ $name =~ $reg ]]; then
+    echo "Adding $1-headers and $1-docs to repository..."
+    cp $run_directory_path/$1/$1-headers-$2*.pkg.tar.xz $run_directory_path/$1/$1-docs-$2*.pkg.tar.xz $repo_directory
+    repo-add $repo_directory/$repo_db_file $repo_directory/$1-headers-$2*.pkg.tar.xz $repo_directory/$1-docs-$2*.pkg.tar.xz
+  fi
+}
+
+clean() {
+  echo "Cleaning directory..."
+  rm -rf $run_directory_path/*
+  echo "Done."
+}
+
+clearRepo() {
+  stream=$(tar xOf $repo_directory/$repo_db_file --wildcards */desc | awk -f $awk_rules_file)
+
+  OIFS=$IFS
+  IFS='|'
+  stream="${stream:1}"
+  current=""
+  for package in $stream
+  do
+    IFS='
+'
+    count=0
+    for line in $package; do
+      if [ $count -eq 0 ]; then
+        current="$current $line"
+      else
+        continue
+      fi
+
+      ((count++))
+    done
+
+    IFS='|'
+  done
+  IFS=$OIFS
+
+  cd $repo_directory
+  reg="^($repo_name.).+"
+  for filename in *; do
+    if [[ $filename =~ $reg || $filename = 'sync' || $filename = 'local'  || $filename = 'resources' ]]; then
+      continue
+    fi
+    for package in $current; do
+      if [ $filename = $package ]; then
+        continue 2
+      fi
+    done
+    echo "Removing $filename..."
+    rm $filename
+  done
+
+}
+
+run_directory_path=$(pwd)
+
+# Set parameters
+repo_directory='/mnt/archrepo'
+repo_name='kto'
+repo_db_file=$repo_name.db.tar
+awk_rules_file='../parser.awk'
+#repo_db_file='kto.db.tar' # You can set db name manually here if different
+
+# Main process
+
+if [ "$1" = "" ]; then
+  init
+  checkUpdates
+elif [ "$1" = "add" ]; then
+  init
+  addPackage $2
+elif [ "$1" = "clean" ]; then
+  clean
+elif [ "$1" = "clear" ]; then
+  read -r -p "Are you sure? [y/N] " response
+  response=${response,,}    # tolower
+  if [[ "$response" =~ ^(yes|y)$ ]]; then
+    clearRepo
+  else
+    echo "Cancelling... Perhaps did you mean 'clean', which empties the local build directory?"
+    exit 1
+  fi
+else
+  exit 1
+fi
+
+exit 0