alex/archrepo
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Initial commit for v1

Browse Source
This commit is contained in:
Alexandre CATTEAU
2022-02-13 12:18:26 +01:00
parent 1dc4a30084
commit 727392393e
4 changed files with 287 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
README.md
View File

@@ -1,12 +1,10 @@
# ArchRepo
## Version 2
### Past versions
* Version 1 is the current version, running state-fully in a dedicated VM, which uses NFS to mount the repository (and
thus works only on a local network)
## Version 1
### Introduction
Version 2 targets an all-new architecture:
* written in Python
* running in a container
* potentially a stateless and ephemeral container
In a nutshell:
* this script is run punctually, on a dedicated Arch VM;
* Proces:
* it takes all packages in the repository;
* checks online if a new version is available
* if so build it and then add it to the repository

17
parser.awk Normal file
View File

@@ -0,0 +1,17 @@
BEGIN {
printit = 0
}
{
if (printit == 1) {
print
printit = 0
}
if (/%FILENAME%/) {
printf "|"
printit = 1
}
if (/%NAME%/ || /%VERSION%/) {
printit = 1
}
}

80
update-repo.py
View File

@@ -1,80 +0,0 @@
# KTO ArchRepo update script
# version 2.0
# The purpose of this script is to automate the maintenance of a custom pacman (Arch Linux) repository, whose goal is to
# provide already built packages of AUR software.
# This version 2 aims at 2 major evolutions:
# * manage dependencies (hence Python and object-oriented programming);
# * run this script in a stateless Arch container, rather than a full VM as in version 1.
# Additionaly, we may like to manage working directory in a better way (check available storage space, and existing
# directories).
# This script should NOT be run as root. Or should it?
# * We are in a stateless container, but I don't think we can run makepkg as root...
# Imports
import os
# Class for AUR package
class Package:
def __init__(self, name):
self.name = name
# + version, dependencies (which should contain version as well)
def parseDb(db_path):
# here we should use AWK to parse our repo database file and populate the collection a collection of packages to build
# that we then return
def checkUpdates(packages_to_build): # or checkDependencies?
# here we should check for all packages in packages_to_build if they need an update and:
# * remove those which do not from the collection
# * check (recursively) for all these packages_to_build if they have dependencies
# * if AUR dependencies, add them to this collection
# * if Arch dependencies, add them to another collection arch_packages_to_install
def installArchPackages(packages):
# packages is a collection of packages (objects or names?) to install
os.system("sudo pacman -Syu " + packages.toSpaceString()) # [].toSpaceString() being to define...
def buildAurPackage(package):
# Fetch package (git clone)
# cd to directory
os.system("makepkg -s --no-confirm --no-progress-bar")
def addPackageToRepo(package, repo):
os.system("cp...")
os.system("repo-add...")
def sortByDependencies():
# packages_to_build should be sorted by dependencies here
# packages that are dependencies should be tagged adequately
def init():
# Check for main dependencies:
deps = ['git', 'awk', 'package-query']
installArchPackages(deps)
# This function is a "main" function
def updateRepo():
init()
packages_to_build = parseDb(db_path)
checkUpdates() # ideally, this should directly edit packages_to_build AND arch_packages_to_install (I believe we can return tuples in Python)
installArchPackages(arch_packages_to_install)
sortByDependencies()
for package in packages_to_build:
buildAurPackage(package)
addPackageToRepo(package, repo)
if package.isDependency:
installArchPackages([package])
def newPackage(package):
init()
checkUpdates([package])
installArchPackages(arch_packages_to_install)
sortByDependencies()
for package in packages_to_build:
buildAurPackage(package)
addPackageToRepo(package, repo)
if package.isDependency:
installArchPackages([package])

263
update-repo.sh Executable file
View File

@@ -0,0 +1,263 @@
#!/bin/bash
# The purpose of this script is to automate the maintenance of a custom pacman (Arch Linux) repository, whose goal is to
# provide package-ready AUR softs.
# This script should NOT be run as root.
# Dependencies
# Make sure that AWK rules file is present, readable and correctly set in this script's parameters
# Make sure the following packages are installed before running this script:
# package-query aurutils git
# Functions
##################################################################
ascii_frag() {
expr match "$1" "\([^[:digit:]]*\)"
}
ascii_remainder() {
expr match "$1" "[^[:digit:]]*\(.*\)"
}
numeric_frag() {
expr match "$1" "\([[:digit:]]*\)"
}
numeric_remainder() {
expr match "$1" "[[:digit:]]*\(.*\)"
}
# return 1 for $1 > $2
# return 2 for $1 < $2
# return 0 for equal
vercomp() {
local WORK1="$1"
local WORK2="$2"
local NUM1="", NUM2="", ASCII1="", ASCII2=""
while true; do
ASCII1=`ascii_frag "${WORK1}"`
ASCII2=`ascii_frag "${WORK2}"`
WORK1=`ascii_remainder "${WORK1}"`
WORK2=`ascii_remainder "${WORK2}"`
if [ "${ASCII1}" \> "${ASCII2}" ]; then
return 1
elif [ "${ASCII1}" \< "${ASCII2}" ]; then
return 2
fi
NUM1=`numeric_frag "${WORK1}"`
NUM2=`numeric_frag "${WORK2}"`
WORK1=`numeric_remainder "${WORK1}"`
WORK2=`numeric_remainder "${WORK2}"`
if [ -z "${NUM1}" -a -z "${NUM2}" ]; then
return 0
elif [ -z "${NUM1}" -a -n "${NUM2}" ]; then
return 2
elif [ -n "${NUM1}" -a -z "${NUM2}" ]; then
return 1
fi
if [ "${NUM1}" -gt "${NUM2}" ]; then
return 1
elif [ "${NUM1}" -lt "${NUM2}" ]; then
return 2
fi
done
}
##################################################################
init() {
# Only run if the user is not root
if [[ $USER = 'root' ]] ; then
echo "You cannot run this script as root!"
exit 1
fi
if [ -n "$(ls $run_directory_path)" ] ; then
echo "The running directory is not empty!"
exit 1
fi
# TODO Check if required packages are installed
echo "ArchRepo update script"
}
checkUpdates() {
echo "Checking for updates:"
upgraded_packages=""
stream=$(tar xOf $repo_directory/$repo_db_file --wildcards */desc | awk -f $awk_rules_file)
OIFS=$IFS
IFS='|'
stream="${stream:1}"
for package in $stream
do
IFS='
'
count=0
for line in $package; do
if [ $count -eq 0 ]; then
filename=$line
elif [ $count -eq 1 ]; then
name=$line
elif [ $count -eq 2 ]; then
version=$line
fi
((count++))
done
reg="(linux-lts).+-((docs)|(headers))"
if [[ $name =~ $reg ]]; then
echo -e "\nSkipping $name ($version, default exception)."
continue 1
fi
exlist=""
if echo $exlist | grep -w $name > /dev/null; then
echo -e "\nSkipping $name ($version, temporary exception)."
continue 1
fi
echo -e "\nChecking $name ($version)..."
aur_version=$(package-query -A -f %v $name)
vercomp $aur_version $version
result=$?
if [[ $result -eq 1 ]]; then
echo "New version for $name: $version -> $aur_version"
createPackage $name
addPackageToRepo $name $aur_version
upgraded_packages="$upgraded_packages $name"
fi
IFS='|'
done
IFS=$OIFS
echo "All packages processed."
if [[ $upgraded_packages != "" ]]; then
echo "The following packages were upgraded:"
echo $upgraded_packages
fi
}
addPackage() {
createPackage $1
addPackageToRepo $1
}
createPackage() { # Create package $1
echo "Fetching $1 from AUR..."
aur fetch $1 # TODO This could be replaced by a git clone or even a wget+tar thanks to package-query
echo "Making $1 package..."
cd $1
# if linux-lts{414,49} then download kernel source from local repo (saves ~100MB each time)
# WARNING this onlys works because we already have said sources AND because PKBUILD follow this particular line
# (not the same in 419 nor 54)
reg="(linux-lts).+"
if [[ $1 =~ $reg ]]; then
sed -i -e "s/\"https:\/\/www.kernel.org\/pub\/linux\/kernel\/v4.x\/\${_srcname}.tar.xz\"/\"https:\/\/archlinux.kto.black\/resources\/\${_srcname}.tar.xz\"/g" ./PKGBUILD
fi
makepkg -s --noconfirm --noprogressbar
cd $run_directory_path
}
addPackageToRepo() { # Add package $1
echo "Adding $1 to repository..."
cp $run_directory_path/$1/$1-$2*.pkg.tar.xz $repo_directory
repo-add $repo_directory/$repo_db_file $repo_directory/$1-$2*.pkg.tar.xz
reg="(linux-lts).+"
if [[ $name =~ $reg ]]; then
echo "Adding $1-headers and $1-docs to repository..."
cp $run_directory_path/$1/$1-headers-$2*.pkg.tar.xz $run_directory_path/$1/$1-docs-$2*.pkg.tar.xz $repo_directory
repo-add $repo_directory/$repo_db_file $repo_directory/$1-headers-$2*.pkg.tar.xz $repo_directory/$1-docs-$2*.pkg.tar.xz
fi
}
clean() {
echo "Cleaning directory..."
rm -rf $run_directory_path/*
echo "Done."
}
clearRepo() {
stream=$(tar xOf $repo_directory/$repo_db_file --wildcards */desc | awk -f $awk_rules_file)
OIFS=$IFS
IFS='|'
stream="${stream:1}"
current=""
for package in $stream
do
IFS='
'
count=0
for line in $package; do
if [ $count -eq 0 ]; then
current="$current $line"
else
continue
fi
((count++))
done
IFS='|'
done
IFS=$OIFS
cd $repo_directory
reg="^($repo_name.).+"
for filename in *; do
if [[ $filename =~ $reg || $filename = 'sync' || $filename = 'local' || $filename = 'resources' ]]; then
continue
fi
for package in $current; do
if [ $filename = $package ]; then
continue 2
fi
done
echo "Removing $filename..."
rm $filename
done
}
run_directory_path=$(pwd)
# Set parameters
repo_directory='/mnt/archrepo'
repo_name='kto'
repo_db_file=$repo_name.db.tar
awk_rules_file='../parser.awk'
#repo_db_file='kto.db.tar' # You can set db name manually here if different
# Main process
if [ "$1" = "" ]; then
init
checkUpdates
elif [ "$1" = "add" ]; then
init
addPackage $2
elif [ "$1" = "clean" ]; then
clean
elif [ "$1" = "clear" ]; then
read -r -p "Are you sure? [y/N] " response
response=${response,,} # tolower
if [[ "$response" =~ ^(yes|y)$ ]]; then
clearRepo
else
echo "Cancelling... Perhaps did you mean 'clean', which empties the local build directory?"
exit 1
fi
else
exit 1
fi
exit 0