diff --git a/README.md b/README.md index e2ab554..d655c7d 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,10 @@ # ArchRepo -## Version 2 - -### Past versions -* Version 1 is the current version, running state-fully in a dedicated VM, which uses NFS to mount the repository (and -thus works only on a local network) +## Version 1 ### Introduction -Version 2 targets an all-new architecture: -* written in Python -* running in a container - * potentially a stateless and ephemeral container +In a nutshell: +* this script is run punctually, on a dedicated Arch VM; +* Proces: + * it takes all packages in the repository; + * checks online if a new version is available + * if so build it and then add it to the repository diff --git a/parser.awk b/parser.awk new file mode 100644 index 0000000..7d12768 --- /dev/null +++ b/parser.awk @@ -0,0 +1,17 @@ +BEGIN { + printit = 0 +} + +{ + if (printit == 1) { + print + printit = 0 + } + if (/%FILENAME%/) { + printf "|" + printit = 1 + } + if (/%NAME%/ || /%VERSION%/) { + printit = 1 + } +} diff --git a/update-repo.py b/update-repo.py deleted file mode 100644 index eb3f8ba..0000000 --- a/update-repo.py +++ /dev/null @@ -1,80 +0,0 @@ -# KTO ArchRepo update script -# version 2.0 - -# The purpose of this script is to automate the maintenance of a custom pacman (Arch Linux) repository, whose goal is to -# provide already built packages of AUR software. - -# This version 2 aims at 2 major evolutions: -# * manage dependencies (hence Python and object-oriented programming); -# * run this script in a stateless Arch container, rather than a full VM as in version 1. -# Additionaly, we may like to manage working directory in a better way (check available storage space, and existing -# directories). - -# This script should NOT be run as root. Or should it? -# * We are in a stateless container, but I don't think we can run makepkg as root... - -# Imports -import os - -# Class for AUR package -class Package: - def __init__(self, name): - self.name = name - # + version, dependencies (which should contain version as well) - -def parseDb(db_path): - # here we should use AWK to parse our repo database file and populate the collection a collection of packages to build - # that we then return - -def checkUpdates(packages_to_build): # or checkDependencies? - # here we should check for all packages in packages_to_build if they need an update and: - # * remove those which do not from the collection - # * check (recursively) for all these packages_to_build if they have dependencies - # * if AUR dependencies, add them to this collection - # * if Arch dependencies, add them to another collection arch_packages_to_install - -def installArchPackages(packages): - # packages is a collection of packages (objects or names?) to install - os.system("sudo pacman -Syu " + packages.toSpaceString()) # [].toSpaceString() being to define... - -def buildAurPackage(package): - # Fetch package (git clone) - # cd to directory - os.system("makepkg -s --no-confirm --no-progress-bar") - -def addPackageToRepo(package, repo): - os.system("cp...") - os.system("repo-add...") - -def sortByDependencies(): - # packages_to_build should be sorted by dependencies here - # packages that are dependencies should be tagged adequately - -def init(): - # Check for main dependencies: - deps = ['git', 'awk', 'package-query'] - installArchPackages(deps) - -# This function is a "main" function -def updateRepo(): - init() - packages_to_build = parseDb(db_path) - checkUpdates() # ideally, this should directly edit packages_to_build AND arch_packages_to_install (I believe we can return tuples in Python) - installArchPackages(arch_packages_to_install) - sortByDependencies() - for package in packages_to_build: - buildAurPackage(package) - addPackageToRepo(package, repo) - if package.isDependency: - installArchPackages([package]) - -def newPackage(package): - init() - checkUpdates([package]) - installArchPackages(arch_packages_to_install) - sortByDependencies() - for package in packages_to_build: - buildAurPackage(package) - addPackageToRepo(package, repo) - if package.isDependency: - installArchPackages([package]) diff --git a/update-repo.sh b/update-repo.sh new file mode 100755 index 0000000..fe3a17f --- /dev/null +++ b/update-repo.sh @@ -0,0 +1,263 @@ +#!/bin/bash + +# The purpose of this script is to automate the maintenance of a custom pacman (Arch Linux) repository, whose goal is to +# provide package-ready AUR softs. +# This script should NOT be run as root. + +# Dependencies +# Make sure that AWK rules file is present, readable and correctly set in this script's parameters +# Make sure the following packages are installed before running this script: +# package-query aurutils git + +# Functions + +################################################################## +ascii_frag() { + expr match "$1" "\([^[:digit:]]*\)" +} + +ascii_remainder() { + expr match "$1" "[^[:digit:]]*\(.*\)" +} + +numeric_frag() { + expr match "$1" "\([[:digit:]]*\)" +} + +numeric_remainder() { + expr match "$1" "[[:digit:]]*\(.*\)" +} + +# return 1 for $1 > $2 +# return 2 for $1 < $2 +# return 0 for equal +vercomp() { + local WORK1="$1" + local WORK2="$2" + local NUM1="", NUM2="", ASCII1="", ASCII2="" + while true; do + ASCII1=`ascii_frag "${WORK1}"` + ASCII2=`ascii_frag "${WORK2}"` + WORK1=`ascii_remainder "${WORK1}"` + WORK2=`ascii_remainder "${WORK2}"` + + if [ "${ASCII1}" \> "${ASCII2}" ]; then + return 1 + elif [ "${ASCII1}" \< "${ASCII2}" ]; then + return 2 + fi + + NUM1=`numeric_frag "${WORK1}"` + NUM2=`numeric_frag "${WORK2}"` + WORK1=`numeric_remainder "${WORK1}"` + WORK2=`numeric_remainder "${WORK2}"` + + if [ -z "${NUM1}" -a -z "${NUM2}" ]; then + return 0 + elif [ -z "${NUM1}" -a -n "${NUM2}" ]; then + return 2 + elif [ -n "${NUM1}" -a -z "${NUM2}" ]; then + return 1 + fi + + if [ "${NUM1}" -gt "${NUM2}" ]; then + return 1 + elif [ "${NUM1}" -lt "${NUM2}" ]; then + return 2 + fi + done +} +################################################################## + +init() { + # Only run if the user is not root + if [[ $USER = 'root' ]] ; then + echo "You cannot run this script as root!" + exit 1 + fi + + if [ -n "$(ls $run_directory_path)" ] ; then + echo "The running directory is not empty!" + exit 1 + fi + + # TODO Check if required packages are installed + + echo "ArchRepo update script" +} + +checkUpdates() { + echo "Checking for updates:" + upgraded_packages="" + stream=$(tar xOf $repo_directory/$repo_db_file --wildcards */desc | awk -f $awk_rules_file) + + OIFS=$IFS + IFS='|' + stream="${stream:1}" + for package in $stream + do + IFS=' +' + count=0 + for line in $package; do + if [ $count -eq 0 ]; then + filename=$line + elif [ $count -eq 1 ]; then + name=$line + elif [ $count -eq 2 ]; then + version=$line + fi + + ((count++)) + done + + reg="(linux-lts).+-((docs)|(headers))" + if [[ $name =~ $reg ]]; then + echo -e "\nSkipping $name ($version, default exception)." + continue 1 + fi + + exlist="" + if echo $exlist | grep -w $name > /dev/null; then + echo -e "\nSkipping $name ($version, temporary exception)." + continue 1 + fi + + echo -e "\nChecking $name ($version)..." + aur_version=$(package-query -A -f %v $name) + + vercomp $aur_version $version + result=$? + if [[ $result -eq 1 ]]; then + echo "New version for $name: $version -> $aur_version" + createPackage $name + addPackageToRepo $name $aur_version + upgraded_packages="$upgraded_packages $name" + fi + + IFS='|' + done + IFS=$OIFS + echo "All packages processed." + + if [[ $upgraded_packages != "" ]]; then + echo "The following packages were upgraded:" + echo $upgraded_packages + fi +} + +addPackage() { + createPackage $1 + addPackageToRepo $1 +} + +createPackage() { # Create package $1 + echo "Fetching $1 from AUR..." + aur fetch $1 # TODO This could be replaced by a git clone or even a wget+tar thanks to package-query + echo "Making $1 package..." + cd $1 + # if linux-lts{414,49} then download kernel source from local repo (saves ~100MB each time) + # WARNING this onlys works because we already have said sources AND because PKBUILD follow this particular line + # (not the same in 419 nor 54) + reg="(linux-lts).+" + if [[ $1 =~ $reg ]]; then + sed -i -e "s/\"https:\/\/www.kernel.org\/pub\/linux\/kernel\/v4.x\/\${_srcname}.tar.xz\"/\"https:\/\/archlinux.kto.black\/resources\/\${_srcname}.tar.xz\"/g" ./PKGBUILD + fi + makepkg -s --noconfirm --noprogressbar + cd $run_directory_path +} + +addPackageToRepo() { # Add package $1 + echo "Adding $1 to repository..." + cp $run_directory_path/$1/$1-$2*.pkg.tar.xz $repo_directory + repo-add $repo_directory/$repo_db_file $repo_directory/$1-$2*.pkg.tar.xz + reg="(linux-lts).+" + if [[ $name =~ $reg ]]; then + echo "Adding $1-headers and $1-docs to repository..." + cp $run_directory_path/$1/$1-headers-$2*.pkg.tar.xz $run_directory_path/$1/$1-docs-$2*.pkg.tar.xz $repo_directory + repo-add $repo_directory/$repo_db_file $repo_directory/$1-headers-$2*.pkg.tar.xz $repo_directory/$1-docs-$2*.pkg.tar.xz + fi +} + +clean() { + echo "Cleaning directory..." + rm -rf $run_directory_path/* + echo "Done." +} + +clearRepo() { + stream=$(tar xOf $repo_directory/$repo_db_file --wildcards */desc | awk -f $awk_rules_file) + + OIFS=$IFS + IFS='|' + stream="${stream:1}" + current="" + for package in $stream + do + IFS=' +' + count=0 + for line in $package; do + if [ $count -eq 0 ]; then + current="$current $line" + else + continue + fi + + ((count++)) + done + + IFS='|' + done + IFS=$OIFS + + cd $repo_directory + reg="^($repo_name.).+" + for filename in *; do + if [[ $filename =~ $reg || $filename = 'sync' || $filename = 'local' || $filename = 'resources' ]]; then + continue + fi + for package in $current; do + if [ $filename = $package ]; then + continue 2 + fi + done + echo "Removing $filename..." + rm $filename + done + +} + +run_directory_path=$(pwd) + +# Set parameters +repo_directory='/mnt/archrepo' +repo_name='kto' +repo_db_file=$repo_name.db.tar +awk_rules_file='../parser.awk' +#repo_db_file='kto.db.tar' # You can set db name manually here if different + +# Main process + +if [ "$1" = "" ]; then + init + checkUpdates +elif [ "$1" = "add" ]; then + init + addPackage $2 +elif [ "$1" = "clean" ]; then + clean +elif [ "$1" = "clear" ]; then + read -r -p "Are you sure? [y/N] " response + response=${response,,} # tolower + if [[ "$response" =~ ^(yes|y)$ ]]; then + clearRepo + else + echo "Cancelling... Perhaps did you mean 'clean', which empties the local build directory?" + exit 1 + fi +else + exit 1 +fi + +exit 0