Add package-query-debug to permanent exceptions

.gitignore

@@ -0,0 +1,2 @@
+variables.conf
+build/*

README.md

@@ -1,14 +1,19 @@
 # ArchRepo
-## Version 2
+## Version 1.1
 
-### Past versions
-* [Version 1](https://gitea.kto.black/alex/archrepo/src/branch/version1) is the previous version, running state-fully in
-a dedicated VM, which uses NFS to mount the repository (and thus works only on a local network)
-  * [Version 1.1](https://gitea.kto.black/alex/archrepo/src/branch/version1.1), currently used, is an adapted version 1
-  with support for a remote repository (via SCP)
+### Version 1
+In a nutshell:
+* this script is run punctually, on a dedicated Arch VM;
+* Process:
+  * it takes all packages in the repository;
+  * checks online if a new version is available
+  * if so build it and then add it to the repository
 
-### Introduction
-Version 2 targets an all-new architecture:
-* written in Python
-* running in a container
-  * potentially a stateless and ephemeral container
+### Version 1.1
+Version 1.1 is an adapted version 1, with remote repository. The idea is to replace NFS by SCP for everything related
+with connection to repository.
+
+We also have a specific clearing script, which should run directly on remote server.
+
+#### TODO
+* Replace `scp` calls by `sftp` (or `rsync`)

clear-repo.sh

@@ -0,0 +1,80 @@
+#!/usr/bin/env bash
+
+# The purpose of this script is to clear our custom pacman repository
+# i.e. remove old versions of packages
+# This script should NOT be run as root.
+
+# Make sure that AWK rules file is present, readable and correctly set in this script's parameters
+
+init() {
+  source variables.conf
+  repo_db_file=$repo_name.db.tar
+  repo_db_filesfile=$repo_name.files.tar
+  awk_rules_file='parser.awk'
+  if [ ! -e $repo_directory/$repo_db_file ] ; then
+    echo "Could not find db file. Exiting..."
+    exit 1
+  fi
+  echo "ArchRepo clearing script"
+}
+
+clearRepo() {
+  if [ "$(uname)" = "OpenBSD" ]; then # TODO This could probably be done better (like with actually checking if tar is GNU...)
+    gnutar="gtar"
+  else
+    gnutar="tar"
+  fi
+  stream=$($gnutar xOf $repo_directory/$repo_db_file --wildcards */desc | awk -f $awk_rules_file)
+
+  OIFS=$IFS
+  IFS='|'
+  stream="${stream:1}"
+  current=""
+  for package in $stream
+  do
+    IFS='
+'
+    count=0
+    for line in $package; do
+      if [ $count -eq 0 ]; then
+        current="$current $line"
+      else
+        continue
+      fi
+
+      ((count++))
+    done
+
+    IFS='|'
+  done
+  IFS=$OIFS
+
+  cd $repo_directory
+  reg="^($repo_name.).+"
+  for filename in *; do
+    if [[ $filename =~ $reg || $filename = 'sync' || $filename = 'local'  || $filename = 'resources' ]]; then
+      continue
+    fi
+    for package in $current; do
+      if [ $filename = $package ]; then
+        continue 2
+      fi
+    done
+    echo "Removing $filename..."
+    rm $filename
+  done
+}
+
+run_directory_path=$(pwd)
+
+# Main process
+
+if [ "$1" = "" ]; then
+  init
+  clearRepo
+else
+  echo "Please call this script without any arguments."
+  exit 1
+fi
+
+exit 0

parser.awk

@@ -0,0 +1,17 @@
+BEGIN {
+  printit = 0
+}
+
+{
+  if (printit == 1) {
+    print
+    printit = 0
+  }
+  if (/%FILENAME%/) {
+    printf "|"
+    printit = 1
+  }
+  if (/%NAME%/ || /%VERSION%/) {
+    printit = 1
+  }
+}

update-repo.py

@@ -1,80 +0,0 @@
-# KTO ArchRepo update script
-# version 2.0
-
-# The purpose of this script is to automate the maintenance of a custom pacman (Arch Linux) repository, whose goal is to
-# provide already built packages of AUR software.
-
-# This version 2 aims at 2 major evolutions:
-# * manage dependencies (hence Python and object-oriented programming);
-# * run this script in a stateless Arch container, rather than a full VM as in version 1.
-# Additionaly, we may like to manage working directory in a better way (check available storage space, and existing
-# directories).
-
-# This script should NOT be run as root. Or should it?
-# * We are in a stateless container, but I don't think we can run makepkg as root...
-
-# Imports
-import os
-
-# Class for AUR package
-class Package:
-  def __init__(self, name):
-    self.name = name
-    # + version, dependencies (which should contain version as well)
-
-def parseDb(db_path):
-  # here we should use AWK to parse our repo database file and populate the collection a collection of packages to build
-  # that we then return
-
-def checkUpdates(packages_to_build): # or checkDependencies?
-  # here we should check for all packages in packages_to_build if they need an update and:
-  # * remove those which do not from the collection
-  # * check (recursively) for all these packages_to_build if they have dependencies
-  #   * if AUR dependencies, add them to this collection
-  #   * if Arch dependencies, add them to another collection arch_packages_to_install
-
-def installArchPackages(packages):
-  # packages is a collection of packages (objects or names?) to install
-  os.system("sudo pacman -Syu " + packages.toSpaceString()) # [].toSpaceString() being to define...
-
-def buildAurPackage(package):
-  # Fetch package (git clone)
-  # cd to directory
-  os.system("makepkg -s --no-confirm --no-progress-bar")
-
-def addPackageToRepo(package, repo):
-  os.system("cp...")
-  os.system("repo-add...")
-
-def sortByDependencies():
-  # packages_to_build should be sorted by dependencies here
-  # packages that are dependencies should be tagged adequately
-
-def init():
-  # Check for main dependencies:
-  deps = ['git', 'awk', 'package-query']
-  installArchPackages(deps)
-
-# This function is a "main" function
-def updateRepo():
-  init()
-  packages_to_build = parseDb(db_path)
-  checkUpdates() # ideally, this should directly edit packages_to_build AND arch_packages_to_install (I believe we can return tuples in Python)
-  installArchPackages(arch_packages_to_install)
-  sortByDependencies()
-  for package in packages_to_build:
-    buildAurPackage(package)
-    addPackageToRepo(package, repo)
-    if package.isDependency:
-      installArchPackages([package])
-
-def newPackage(package):
-  init()
-  checkUpdates([package])
-  installArchPackages(arch_packages_to_install)
-  sortByDependencies()
-  for package in packages_to_build:
-    buildAurPackage(package)
-    addPackageToRepo(package, repo)
-    if package.isDependency:
-      installArchPackages([package])

update-repo.sh

@@ -0,0 +1,226 @@
+#!/usr/bin/env bash
+
+# The purpose of this script is to automate the maintenance of a custom pacman (Arch Linux) repository, whose goal is to
+# provide package-ready AUR softs.
+# This script should NOT be run as root.
+
+# Dependencies
+# Make sure that AWK rules file is present, readable and correctly set in this script's parameters
+# Make sure the following packages are installed before running this script:
+# package-query aurutils git
+
+
+# Functions
+
+##################################################################
+ascii_frag() {
+    expr match "$1" "\([^[:digit:]]*\)"
+}
+
+ascii_remainder() {
+    expr match "$1" "[^[:digit:]]*\(.*\)"
+}
+
+numeric_frag() {
+    expr match "$1" "\([[:digit:]]*\)"
+}
+
+numeric_remainder() {
+    expr match "$1" "[[:digit:]]*\(.*\)"
+}
+
+# return 1 for $1 > $2
+# return 2 for $1 < $2
+# return 0 for equal
+vercomp() {
+    local WORK1="$1"
+    local WORK2="$2"
+    local NUM1="", NUM2="", ASCII1="", ASCII2=""
+    while true; do
+        ASCII1=`ascii_frag "${WORK1}"`
+        ASCII2=`ascii_frag "${WORK2}"`
+        WORK1=`ascii_remainder "${WORK1}"`
+        WORK2=`ascii_remainder "${WORK2}"`
+
+        if [ "${ASCII1}" \> "${ASCII2}" ]; then
+            return 1
+        elif [ "${ASCII1}" \< "${ASCII2}" ]; then
+            return 2
+        fi
+
+        NUM1=`numeric_frag "${WORK1}"`
+        NUM2=`numeric_frag "${WORK2}"`
+        WORK1=`numeric_remainder "${WORK1}"`
+        WORK2=`numeric_remainder "${WORK2}"`
+
+        if [ -z "${NUM1}" -a -z "${NUM2}" ]; then
+            return 0
+        elif [ -z "${NUM1}" -a -n "${NUM2}" ]; then
+            return 2
+        elif [ -n "${NUM1}" -a -z "${NUM2}" ]; then
+            return 1
+        fi
+
+        if [ "${NUM1}" -gt "${NUM2}" ]; then
+            return 1
+        elif [ "${NUM1}" -lt "${NUM2}" ]; then
+            return 2
+        fi
+    done
+}
+##################################################################
+
+init() {
+  # Only run if the user is not root
+  if [[ $USER = 'root' ]] ; then
+    echo "You cannot run this script as root!"
+    exit 1
+  fi
+
+  if [ -n "$(ls $run_directory_path)" ] ; then
+    echo "The running directory is not empty!"
+    exit 1
+  fi
+
+  source ../variables.conf
+  repo_db_file=$repo_name.db.tar
+  repo_db_filesfile=$repo_name.files.tar
+  awk_rules_file='../parser.awk'
+  eval "$(ssh-agent)"
+  ssh-add $ssh_key
+
+  # TODO Check if required packages are installed
+
+  echo "ArchRepo update script"
+}
+
+checkUpdates() {
+  echo "Checking for updates:"
+  upgraded_packages=""
+  scp $ssh_options $remote_repository/$repo_db_file $remote_repository/$repo_db_filesfile $run_directory_path/
+  stream=$(tar xOf $run_directory_path/$repo_db_file --wildcards */desc | awk -f $awk_rules_file)
+
+  OIFS=$IFS
+  IFS='|'
+  stream="${stream:1}"
+  for package in $stream
+  do
+    IFS='
+'
+    count=0
+    for line in $package; do
+      if [ $count -eq 0 ]; then
+        filename=$line
+      elif [ $count -eq 1 ]; then
+        name=$line
+      elif [ $count -eq 2 ]; then
+        version=$line
+      fi
+
+      ((count++))
+    done
+
+    reg="(linux-lts).+-((docs)|(headers))"
+    if [[ $name =~ $reg ]]; then
+      echo -e "\nSkipping $name ($version, default exception)."
+      continue 1
+    fi
+
+    reg="(kodi-addon-game).+-((debug))"
+    if [[ $name =~ $reg ]]; then
+      echo -e "\nSkipping $name ($version, default exception)."
+      continue 1
+    fi
+
+    if echo $exlist | grep -w $name > /dev/null; then
+      echo -e "\nSkipping $name ($version, temporary exception)."
+      continue 1
+    fi
+
+    reg="(package-query-debug)"
+    if [[ $name =~ $reg ]]; then
+      echo -e "\nSkipping $name ($version, default exception)."
+      continue 1
+    fi
+
+    echo -e "\nChecking $name ($version)..."
+    aur_version=$(package-query -A -f %v $name)
+
+    vercomp $aur_version $version
+    result=$?
+    if [[ $result -eq 1 ]]; then
+      echo "New version for $name: $version -> $aur_version"
+      IFS=$OIFS
+      createPackage $name
+      addPackageToRepo $name $aur_version
+      upgraded_packages="$upgraded_packages $name"
+    fi
+
+    IFS='|'
+  done
+  IFS=$OIFS
+  echo "All packages processed."
+
+  if [[ $upgraded_packages != "" ]]; then
+    echo "The following packages were upgraded:"
+    echo $upgraded_packages
+  fi
+}
+
+addPackage() {
+  createPackage $1
+  addPackageToRepo $1
+}
+
+createPackage() { # Create package $1
+  echo "Fetching $1 from AUR..."
+  aur fetch $1 # TODO This could be replaced by a git clone or even a wget+tar thanks to package-query
+  echo "Making $1 package..."
+  cd $1
+  makepkg -s --noconfirm --noprogressbar
+  cd $run_directory_path
+}
+
+# TODO: try manually: can we update db file before sending to remote repo?
+addPackageToRepo() { # Add package $1
+  echo "Adding $1 to repository..."
+  scp $ssh_options $run_directory_path/$1/$1-$2*.pkg.tar.* $remote_repository/
+  repo-add $repo_db_file $run_directory_path/$1/$1-$2*.pkg.tar.*
+  reg="(linux-lts).+"
+  if [[ $name =~ $reg ]]; then
+    echo "Adding $1-headers and $1-docs to repository..."
+    repo-add $repo_db_file $run_directory_path/$1/$1-headers-$2*.pkg.tar.* $run_directory_path/$1/$1-docs-$2*.pkg.tar.*
+    scp $ssh_options $run_directory_path/$1/$1-headers-$2*.pkg.tar.* $run_directory_path/$1/$1-docs-$2*.pkg.tar.* $remote_repository/
+  fi
+  scp $ssh_options $run_directory_path/$repo_db_file $run_directory_path/$repo_db_file.old $run_directory_path/$repo_db_filesfile $run_directory_path/$repo_db_filesfile.old $remote_repository/
+}
+
+clean() {
+  echo "Cleaning directory..."
+  rm -rf $run_directory_path/*
+  echo "Done."
+}
+
+run_directory_path=$(pwd)
+
+# Main process
+
+if [ "$1" = "" ]; then
+  init
+  checkUpdates
+elif [ "$1" = "add" ]; then
+  init
+  scp $ssh_options $remote_repository/$repo_db_file $remote_repository/$repo_db_filesfile $run_directory_path/
+  for arg in "$@"; do
+    if [[ $arg = "add" ]]; then
+      continue
+    fi
+    addPackage $arg
+  done
+elif [ "$1" = "clean" ]; then
+  clean
+else
+  exit 1
+fi
+
+exit 0

variables.conf.template

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+# Copy this file as variables.conf, with the correct values
+
+# For both update and clear scripts:
+repo_name=""
+
+# For update script:
+remote_repository=""
+ssh_key=""
+ssh_options="-i $ssh_key"
+exlist=""
+
+# For clear script:
+repo_directory=""