Small improvements to release script.

Add usage information and todo list to release script.
Remove old release scripts.

extra/scripts/create-wifibox-updates-dir.sh

@@ -1,188 +0,0 @@
-#!/bin/sh
-
-# NOTE: this script generates an index based on images found in the target directory.
-# So make sure it contains all images ever released (unless you want to actually remove them).
-# If this is not the case, first create a mirror of doodle3d.com/updates/images.
-
-#prevent being run as root (which is dangerous)
-if [ "$(id -u)" == "0" ]; then
-   echo "Don't run this script as root, it is potentially dangerous." 1>&2
-   exit 1
-fi
-
-
-# expects path of file to return the size of
-fileSize() {
-	stat -f %z ${1}
-}
-
-# expects arguments: version, devType, sysupgrade|factory
-# image name will be: '${IMAGE_BASENAME}-<version>-<devType>-<sysupgrade|factory>.bin'
-constructImageName() {
-	if [ $# -lt 3 ]; then echo "incorrect usage of constructImageName()"; exit 1; fi
-	echo "${IMAGE_BASENAME}-${1}-${2}-${3}.bin"
-}
-
-# expects arguments: basePath (where the files are), version, devType
-generateIndexEntry() {
-	if [ $# -lt 3 ]; then echo "incorrect usage of generateIndexEntry()"; exit 1; fi
-
-	sysupgrade_out_basename=`constructImageName ${2} ${3} sysupgrade`
-	factory_out_basename=`constructImageName ${2} ${3} factory`
-	sysupgrade_out_file=${1}/${sysupgrade_out_basename}
-	factory_out_file=${1}/${factory_out_basename}
-
-	sysupgrade_filesize=`fileSize ${sysupgrade_out_file}`
-	factory_filesize=`fileSize ${factory_out_file}`
-	sysupgrade_md5sum=`md5 -q ${sysupgrade_out_file}`
-	factory_md5sum=`md5 -q ${factory_out_file}`
-	echo "Version: ${2}" >> $IMG_INDEX_FILE
-	echo "Files: ${sysupgrade_out_basename}; ${factory_out_basename}" >> $IMG_INDEX_FILE
-	echo "FileSize: ${sysupgrade_filesize}; ${factory_filesize}" >> $IMG_INDEX_FILE
-	echo "MD5: ${sysupgrade_md5sum}; ${factory_md5sum}" >> $IMG_INDEX_FILE
-}
-
-
-OPENWRT_BASE=.
-PKG_SRC_DIR=$OPENWRT_BASE/bin/ar71xx/packages
-PKG_DEST_SUBPATH=updates
-MAKE_INDEX_SCRIPT=$OPENWRT_BASE/scripts/ipkg-make-index.sh
-INDEX_FILE=Packages
-INDEX_GZ_FILE=Packages.gz
-DEVICE_TYPES="tl-mr3020 tl-wr703"
-MAX_GOOD_IMAGE_SIZE=3500000
-IMAGE_BASENAME="doodle3d-wifibox"
-
-COMPRESS_RESULT=0
-PKG_DEST_BASE=.
-
-
-for arg in "$@"; do
-	case $arg in
-		-h)
-			echo "This script creates a directory with wifibox and ultifi ipk files found in the openWrt build environment."
-			echo "The feed dir is called $PKG_DEST_DIR and will be created in the current directory. (currently `pwd`)"
-			echo "If specified, the -z option also compresses the result for easier transfer to a webserver."
-			exit
-			;;
-		-z)
-			COMPRESS_RESULT=1
-			;;
-		-*)
-			echo "Unrecognized option '$arg'"
-			exit 1
-			;;
-		*)
-			PKG_DEST_BASE=$arg
-			;;
-	esac
-done
-
-grep "^This is the buildsystem for the OpenWrt Linux distribution\.$" README >/dev/null 2>&1
-if [ $? -ne 0 ]; then
-	echo "Please run this script from the Openwrt build root (on OSX this is probably an image mounted under /Volumes)."
-	exit 1
-fi
-
-
-#determine the wifibox root path
-my_rel_dir=`dirname $0`
-pushd "$my_rel_dir" > /dev/null
-WIFIBOX_DIR="`pwd`/../.."
-popd > /dev/null
-
-FW_VERSION=`cat $WIFIBOX_DIR/src/FIRMWARE-VERSION`
-echo "Compiling firmware update files for version ${FW_VERSION}"
-
-
-#setup paths
-PKG_DEST_DIR=$PKG_DEST_BASE/$PKG_DEST_SUBPATH
-PKG_FEED_DIR=$PKG_DEST_DIR/feed
-PKG_IMG_DIR=$PKG_DEST_DIR/images
-IMG_INDEX_FILE=$PKG_IMG_DIR/wifibox-image.index
-
-if [ ! -d $PKG_DEST_DIR ]; then mkdir -p $PKG_DEST_DIR; fi
-echo "Using $PKG_DEST_DIR as target directory"
-
-
-#clear directory and copy package files
-if [ ! -d $PKG_FEED_DIR ]; then mkdir $PKG_FEED_DIR; fi
-cp $PKG_SRC_DIR/wifibox*.ipk $PKG_FEED_DIR
-cp $PKG_SRC_DIR/doodle3d-client*.ipk $PKG_FEED_DIR
-cp $PKG_SRC_DIR/print3d*.ipk $PKG_FEED_DIR
-cp $PKG_SRC_DIR/ultifi*.ipk $PKG_FEED_DIR
-rm -f $PKG_FEED_DIR/$INDEX_FILE
-rm -f $PKG_FEED_DIR/$INDEX_GZ_FILE
-
-
-#copy and rename images
-if [ ! -d $PKG_IMG_DIR ]; then mkdir $PKG_IMG_DIR; fi
-rm -f $IMG_INDEX_FILE
-for devtype in $DEVICE_TYPES; do
-	IMG_SRC_PATH=$OPENWRT_BASE/bin/ar71xx
-	if [ -f $IMG_SRC_PATH/openwrt-ar71xx-generic-${devtype}-v1-squashfs-sysupgrade.bin ]; then
-		sysupgrade_name=$IMG_SRC_PATH/openwrt-ar71xx-generic-${devtype}-v1-squashfs-sysupgrade.bin
-		factory_name=$IMG_SRC_PATH/openwrt-ar71xx-generic-${devtype}-v1-squashfs-factory.bin
-		sysupgrade_size=`fileSize $sysupgrade_name`
-		factory_size=`fileSize $factory_name`
-
-		echo "Copying images for device '${devtype}' (sysupgrade size: ${sysupgrade_size}, factory size: ${factory_size})"
-		cp $sysupgrade_name $PKG_IMG_DIR/`constructImageName ${FW_VERSION} ${devtype} sysupgrade`
-		cp $factory_name $PKG_IMG_DIR/`constructImageName ${FW_VERSION} ${devtype} factory`
-
-		if [ $sysupgrade_size -gt $MAX_GOOD_IMAGE_SIZE ]; then
-			echo "WARNING: the sysupgrade image is larger than $MAX_GOOD_IMAGE_SIZE bytes, which probably means it will cause read/write problems when flashed to a device"
-		fi
-	fi
-done
-
-
-# ok this is ugly, but at least it generates a complete index (the loop assumes for each
-# sysupgrade image it finds, there is also a factory counterpart)
-for file in $PKG_IMG_DIR/${IMAGE_BASENAME}-*-sysupgrade.bin; do
-	basefile=`basename ${file}`
-	echo "Considering $basefile (${file})"
-	# sorry for the shell magic
-	devtype=${basefile:`expr ${#IMAGE_BASENAME} + 1`}
-	version=${devtype//-*}
-	devtype=${devtype#*-}
-	devtype=${devtype%-*}
-	generateIndexEntry $PKG_IMG_DIR $version $devtype >> $IMG_INDEX_FILE
-done
-
-
-# NOTE: the aliasing construct in the indexing script does not work (and even then, the md5 command defaults to a different output format), so we hack around it here.
-MD5_HACK_ENABLED=0
-which md5sum >/dev/null 2>&1
-if [ $? -eq 1 ]; then
-	MD5_HACK_ENABLED=1
-	TEMPBIN_DIR=/tmp/tempbin23QQDBR
-	mkdir $TEMPBIN_DIR
-
-	cat <<EOF > $TEMPBIN_DIR/md5sum
-`type -p md5` -q \$1
-EOF
-
-	chmod +x $TEMPBIN_DIR/md5sum
-	PATH=$PATH:$TEMPBIN_DIR
-fi
-
-#this cwd juggling is required to have the package indexer generate correct paths (i.e. no paths) in the Packages file
-OPENWRT_DIR=`pwd`
-pushd $PKG_FEED_DIR > /dev/null
-$OPENWRT_DIR/$MAKE_INDEX_SCRIPT . > $PKG_FEED_DIR/$INDEX_FILE
-popd > /dev/null
-
-if [ $MD5_HACK_ENABLED -eq 1 ]; then
-	rm $TEMPBIN_DIR/md5sum
-	rmdir $TEMPBIN_DIR
-fi
-
-gzip -c $PKG_FEED_DIR/$INDEX_FILE > $PKG_FEED_DIR/$INDEX_GZ_FILE
-
-
-if [ $COMPRESS_RESULT -eq 1 ]; then
-	cd $PKG_DEST_BASE
-	echo "Compressing generated package directory..."
-	tar czvf "doodle3d-wifibox-update-dist.tgz" $PKG_DEST_SUBPATH
-fi

extra/scripts/publish-wifibox-release.lua

@@ -1,7 +1,43 @@
 #!/usr/bin/env lua
 --#!/usr/bin/env lua -l strict
 
---TODO: replace prints with D() function from update manager and other slightly smarter mechanisms?
+-- This script creates a new release by copying openwrt image files and release notes to a local
+-- directory and updating the relevant index file with a new entry. This directory is
+-- then synchronized to the release repository online.
+--
+-- USAGE:
+-- The only dependency of this script is the penlight library, which can be installed using
+-- LuaRocks (http://luarocks.org/) as follows: 'sudo luarocks install penlight'.
+-- The script must be run from within the openwrt build root. It will automatically locate
+-- the Doodle3D repo's. Index files are fetched from the online repository.
+-- For synchronizing, rsync must have passwordless SSH access to the server, for a
+-- guide, see: http://www.linuxproblem.org/art_9.html.
+-- Some basic sanity checks are built in (unique version, updated release notes, 'clean' openwrt config)
+-- but lots others are still missing (mainly: clean git repo's, freshly built images).
+-- Before anything is actually uploaded, you will be asked if that's really what you want to do.
+-- It might be wise to make a backup on the server before updating it, there's a script
+-- to do this on the server: '~/backup-updates-dir.sh'.
+--
+-- To play around with or improve on this script, use and modify the variables 'SERVER_HOST'
+-- and 'SERVER_PATH' below to point to your machine (assuming you have a webserver running there).
+-- Also uncomment and modify UPDATER_BASE_URL. You will have to init the local 'repo' with at
+-- least empty index files ('wifibox-image.index' and 'wifibox-image.beta.index'), or you
+-- could of course mirror the online repository.
+--
+-- TODO (in random order):
+-- * (feature) command-line arguments: overrides, verbosity, allow local mirroring, clear local cache dir, etc.
+-- * (feature) automatically create a backup of the online repo (there's already a script fir this, as mentioned above)
+-- * (feature) check whether git repo's are clean and on correct branch
+-- * (feature) allow local mirroring with a reverse rsync command and rebuilding the indexes
+--   - update manager 'cache' should then be enabled to prevent fetchIndexTable from downloading files
+-- * (feature) automatically (re)build openwrt to ensure it is up to date?
+-- * (feature) update package feed (requires a local mirror for the feed indexing script)
+--   - in this case sanity checks must also be run on package versions/revisions
+-- * (feature) automatically tag (and merge?) git commits?
+-- * (feature) execute as dry-run by default so changes can be reviewed?
+-- * (refactor) rename awkward vars/funcs regarding entries, versions and caches...
+-- * (refactor) replace function arguments 'includeBetas' with a set function like setUseCache to improve readability
+-- * (refactor) replace prints with D() function from update manager or other slightly smarter mechanisms?
 
 local function ERR(msg) print(msg) end
 
@@ -22,8 +58,14 @@ local lfs = require('lfs') -- assume this exists since it's required by penlight
 
 --local SERVER_HOST = 'localhost'
 --local SERVER_PATH = '~USERDIR/public_html/wifibox/updates'
+--local UPDATER_BASE_URL = 'http://localhost/~USERDIR/wifibox/updates'
 local SERVER_HOST = 'doodle3d.com'
 local SERVER_PATH = 'doodle3d.com/DEFAULT/updates'
+--- SERVER_HOST and SERVER_PATH are used by rsync to merge the local working directory
+-- back into the online repository (requires functioning public key SSH access).
+-- UPDATER_BASE_URL is used by the d3d-updater script to download the index files
+-- (over HTTP), it defaults to the doodle3d.com online repo so it should only be
+-- used for development purposes.
 
 local D3D_REPO_FIRMWARE_NAME = 'doodle3d-firmware'
 local D3D_REPO_CLIENT_NAME = 'doodle3d-client'
@@ -134,12 +176,12 @@ end
 -- TODO: pass table to functions to fill in? if they all return either true or nil+msg, that could be used for display of ok/msg
 -- returns true on success, false on error, and displays meaningful messages
 --local function runCheck(msg, processFunc)
---	io.stdout:write(msg .. "... ")
+--	io.write(msg .. "... ")
 --	return processFunc(--[[ hmm ]]--)
 --end
 
 local function runAction(actMsg, errMsg, ev, func)
-	io.stdout:write("* " .. actMsg .. "...")
+	io.write("* " .. actMsg .. "...")
 	local rv,err = func()
 	if not rv then
 		if err then print("Error: " .. errMsg .. " (" .. err .. ")")
@@ -185,7 +227,7 @@ end
 local function prepare()
 	local msg = nil
 
-	io.stdout:write("* Checking if working directory is the OpenWrt root... ")
+	io.write("* Checking if working directory is the OpenWrt root... ")
 	local isOpenWrtRoot = detectOpenWrtRoot()
 	if isOpenWrtRoot then
 		paths.wrt = pl.path.currentdir()
@@ -195,7 +237,7 @@ local function prepare()
 		return nil
 	end
 
-	io.stdout:write("* Looking for Doodle3D feed path... ")
+	io.write("* Looking for Doodle3D feed path... ")
 	local d3dFeed,msg = getWifiboxFeedRoot('feeds.conf')
 	if d3dFeed then
 		print("found " .. d3dFeed)
@@ -213,7 +255,7 @@ local function prepare()
 		--paths.cache = pl.app.appfile('')
 		paths.cache = '/tmp/d3d-release-dir'
 	end
-	io.stdout:write("* Attempting to use " .. paths.cache .. " as cache dir... ")
+	io.write("* Attempting to use " .. paths.cache .. " as cache dir... ")
 	local rv,msg = pl.dir.makepath(paths.cache)
 	if not rv then
 		print("could not create path (" .. msg .. ").")
@@ -236,6 +278,19 @@ local function prepare()
 		print("ok")
 	end
 
+		-- initialize update manager script
+	um.setUseCache(false)
+	um.setVerbosity(1)
+	um.setCachePath(imageCachePath())
+	if type(UPDATER_BASE_URL) == 'string' and UPDATER_BASE_URL:len() > 0 then
+		print("* Using updater base URL: '" .. UPDATER_BASE_URL .. "'")
+		um.setBaseUrl(UPDATER_BASE_URL)
+	else
+		print("* Using updater base URL: d3d-updater default")
+	end
+
+	print("* Using rsync server remote: '" .. SERVER_HOST .. "/" .. SERVER_PATH .. "'")
+
 	return true
 end
 
@@ -357,7 +412,7 @@ local function checkWrtConfig()
 	local wrtConfigPath = pl.path.tmpname()
 	--print("diffonfig output file: " .. wrtConfigPath)
 
-	local rv,ev = pl.utils.execute('./scripts/diffconfig.sh > "' .. wrtConfigPath .. '"')
+	local rv,ev = pl.utils.execute('./scripts/diffconfig.sh > "' .. wrtConfigPath .. '" 2> /dev/null')
 	if not rv then return nil,"could not run diffconfig script (exit status: " .. ev .. ")" end
 
 	local _,rv,output = pl.utils.executeex('diff "' .. wrtConfigPath .. '" "' .. goodConfigPath .. '"')
@@ -387,11 +442,6 @@ local function main()
 
 	if not prepare() then quit(1) end
 
-	-- initialize update manager script
-	um.setUseCache(false)
-	um.setVerbosity(1)
-	um.setCachePath(imageCachePath())
-	--um.setBaseUrl('http://localhost/~USERDIR/wifibox/updates')
 
 	local newVersion,msg = collectLocalInfo()
 	if not newVersion then
@@ -445,7 +495,7 @@ local function main()
 		return copyImages(newVersion)
 	end)
 
-	io.stdout:write("* Building package feed directory...")
+	io.write("* Building package feed directory...")
 	print("skipped - not implemented")
 --	runAction("Building package feed directory", "failed", 5, buildFeedDir)
 

extra/scripts/release-wifibox-updates-dir.sh

@@ -1,54 +0,0 @@
-#!/bin/sh
-
-# This script copies the packages feed & images directory to ~/Sites (e.g. for use with XAMPP or the like).
-# Using the -u option, it can also upload to doodle3d.com/updates (make sure ssh automatically uses the correct username, or change the rsync command below).
-# Modify WIFIBOX_BASE_DIR to point to your wifibox directory tree.
-
-WIFIBOX_BASE_DIR=~/Files/_devel/eclipse-workspace/doodle3d/doodle3d-firmware
-DEST_DIR=~/public_html/wifibox
-UPDATES_DIR=updates
-BASE_URL=doodle3d.com
-
-OPTIONS=$DEST_DIR
-UPLOAD_FILES=0
-
-for arg in "$@"; do
-	case $arg in
-		-h)
-			echo "This script calls 'create-wifibox-updates-dir.sh' to generate feed/image directories in $DEST_DIR"
-			echo "Use '-z' to also create a compressed file containing the 'updates' directory."
-			echo "Used '-u' to also upload the directory to doodle3d.com/updates"
-			exit
-			;;
-		-z)
-			OPTIONS="$OPTIONS -z"
-			;;
-		-u)
-			UPLOAD_FILES=1
-			;;
-		*)
-			echo "Unrecognized option '$arg'"
-			exit 1
-			;;
-	esac
-done
-
-$WIFIBOX_BASE_DIR/extra/scripts/create-wifibox-updates-dir.sh $OPTIONS
-RETURN_VALUE=$?
-if [ $RETURN_VALUE -ne 0 ]; then
-	echo "create-wifibox-updates-dir.sh returned an error (${RETURN_VALUE}), exiting"
-	exit $RETURN_VALUE
-fi
-
-if [ $UPLOAD_FILES -eq 1 ]; then
-	#UPLOAD_PATH=$BASE_URL:public_html/updates
-	UPLOAD_PATH=$BASE_URL:doodle3d.com/DEFAULT/updates
-
-	cat <<-'EOM' > $DEST_DIR/$UPDATES_DIR/.htaccess
-	Options +Indexes
-EOM
-
-	echo "Uploading files to $UPLOAD_PATH (if you are asked for your password, please add an entry to your ~/.ssh/config and upload your public ssh key)"
-	#options are: recursive, preserve perms, symlinks and timestamps, be verbose and use compression
-	rsync -rpltvz -e ssh --progress $DEST_DIR/$UPDATES_DIR/.htaccess $DEST_DIR/$UPDATES_DIR/* $UPLOAD_PATH
-fi