Merge branch 'feature/logrotate' into develop

Makefile

@@ -66,6 +66,13 @@ define Build/Compile directives
 #	no compilation necessary (although possible with luac?)
 endef
 
+# This information is contained within the ipk file and (at least) used by
+# sysupgrade to determine which files to keep (see `opkg list-changed-conffiles`).
+define Package/wifibox/conffiles
+/etc/config/wifibox
+/etc/logrotate.d/wifibox.conf
+endef
+
 # The $(1) variable represents the root directory on the router running 
 # OpenWrt. The $(INSTALL_DIR) variable contains a command to prepare the install 
 # directory if it does not already exist.  Likewise $(INSTALL_BIN) contains the 
@@ -88,6 +95,7 @@ define Package/wifibox/install
 	#$(INSTALL_DIR) $(1)/etc
 	$(INSTALL_DIR) $(1)/etc/init.d
 	$(INSTALL_DIR) $(1)/etc/config
+	$(INSTALL_DIR) $(1)/etc/logrotate.d
 	$(INSTALL_DIR) $(1)/root/
 	$(INSTALL_DIR) $(1)/root/sketches
 	#$(INSTALL_DIR) $(1)/www
@@ -111,6 +119,7 @@ define Package/wifibox/install
 	$(INSTALL_BIN) $(WIFIBOX_BASE_DIR)/script/dhcpcheck_init $(1)/etc/init.d/dhcpcheck
 	$(INSTALL_BIN) $(WIFIBOX_BASE_DIR)/script/d3dapi $(1)/$(TGT_LUA_DIR_SUFFIX)/script
 	$(INSTALL_BIN) $(WIFIBOX_BASE_DIR)/script/signin.sh $(1)/$(TGT_LUA_DIR_SUFFIX)/script
+	$(CP) $(WIFIBOX_BASE_DIR)/script/logrotate-wifibox.conf $(1)/etc/logrotate.d/wifibox.conf
 	
 	$(CP) $(WIFIBOX_BASE_DIR)/script/wifibox.uci.config $(1)/etc/config/wifibox  # copy base configuration to uci config dir
 	$(CP) $(WIFIBOX_BASE_DIR)/FIRMWARE-VERSION $(1)/etc/wifibox-version

extra/openwrt-build/openwrt-diffconfig-extramini

@@ -22,8 +22,10 @@ CONFIG_PACKAGE_kmod-usb-serial-ftdi=y
 # CONFIG_PACKAGE_libip6tc is not set
 CONFIG_PACKAGE_libiwinfo=y
 CONFIG_PACKAGE_libiwinfo-lua=y
+CONFIG_PACKAGE_libpopt=y
 CONFIG_PACKAGE_liblua=y
 CONFIG_PACKAGE_libuci-lua=y
+CONFIG_PACKAGE_logrotate=y
 CONFIG_PACKAGE_lua=y
 CONFIG_PACKAGE_luafilesystem=y
 # CONFIG_PACKAGE_odhcp6c is not set

extra/openwrt-build/openwrt-diffconfig-mini

@@ -20,8 +20,10 @@ CONFIG_PACKAGE_kmod-usb-serial-ftdi=y
 # CONFIG_PACKAGE_libip6tc is not set
 CONFIG_PACKAGE_libiwinfo=y
 CONFIG_PACKAGE_libiwinfo-lua=y
+CONFIG_PACKAGE_libpopt=y
 CONFIG_PACKAGE_liblua=y
 CONFIG_PACKAGE_libuci-lua=y
+CONFIG_PACKAGE_logrotate=y
 CONFIG_PACKAGE_lua=y
 CONFIG_PACKAGE_luafilesystem=y
 # CONFIG_PACKAGE_odhcp6c is not set

extra/openwrt-build/openwrt-diffconfig-regular

@@ -9,8 +9,10 @@ CONFIG_PACKAGE_kmod-usb-serial=y
 CONFIG_PACKAGE_kmod-usb-serial-ftdi=y
 CONFIG_PACKAGE_libiwinfo=y
 CONFIG_PACKAGE_libiwinfo-lua=y
+CONFIG_PACKAGE_libpopt=y
 CONFIG_PACKAGE_liblua=y
 CONFIG_PACKAGE_libuci-lua=y
+CONFIG_PACKAGE_logrotate=y
 CONFIG_PACKAGE_lua=y
 CONFIG_PACKAGE_luafilesystem=y
 CONFIG_PACKAGE_print3d=y

post-install.sh

@@ -76,11 +76,11 @@ fi
 #preserve saved sketches during firmware update
 echo "/root/sketches" >> $IPKG_INSTROOT/etc/sysupgrade.conf
 
+
+
 ### Finally make sure basic configuration is set correctly
 
-$IPKG_INSTROOT/etc/init.d/wifibox enable
-$IPKG_INSTROOT/etc/init.d/wifibox start
-$IPKG_INSTROOT/etc/init.d/dhcpcheck enable
+LOGROTATE_CRON_LINE="*/3 *   *   *   *   /usr/sbin/logrotate /etc/logrotate.conf"
 
 if [ -z "$IPKG_INSTROOT" ]; then
 	# No installation root, we are being installed on a live box so run uci commands directly.
@@ -105,6 +105,13 @@ if [ -z "$IPKG_INSTROOT" ]; then
 	uci -q delete wifibox.system.loglevel  # remove key used in older versions (<=0.10.8a) if it exists
 	uci commit wifibox
 
+	crontab -l 2> /dev/null | grep logrotate\.conf > /dev/null
+	if [ $? -ne 0 ]; then
+		# add line, method from http://askubuntu.com/a/58582
+		# Note: `crontab -l` will throw an error to stderr because the file does not exist, but that does not matter
+		(crontab -l 2> /dev/null; echo "$LOGROTATE_CRON_LINE" ) | crontab -
+	fi
+
 else
 	# Create a script to setup the system as wifibox, it will be deleted after it has been run, except if it returns > 0.
 
@@ -127,10 +134,23 @@ else
 	uci set wifibox.general.system_log_level='info'
 	uci -q delete wifibox.system.loglevel  # remove key used in older versions (<=0.10.8a) if it exists
 
+	crontab -l 2> /dev/null | grep logrotate\.conf > /dev/null
+	if [ $? -ne 0 ]; then
+		# add line, method from http://askubuntu.com/a/58582
+		# Note: `crontab -l` will throw an error to stderr because the file does not exist, but that does not matter
+		(crontab -l 2> /dev/null; echo "$LOGROTATE_CRON_LINE" ) | crontab -
+	fi
+
 	exit 0
 EOM
 
 	echo "WARNING: WiFiBox network configuration can only be fully prepared when installing on real device"
 fi
 
+$IPKG_INSTROOT/etc/init.d/wifibox enable
+$IPKG_INSTROOT/etc/init.d/wifibox start
+$IPKG_INSTROOT/etc/init.d/dhcpcheck enable
+$IPKG_INSTROOT/etc/init.d/cron enable
+$IPKG_INSTROOT/etc/init.d/cron start
+
 exit 0

src/rest/api/api_info.lua

@@ -23,6 +23,8 @@ local DEFAULT_WIFIBOX_LOG_FILENAME = 'wifibox.log'
 local DEFAULT_WIFIBOX_LOG_FILE = TMP_DIR .. '/' .. DEFAULT_WIFIBOX_LOG_FILENAME
 local WIFIBOX_STDOUT_LOG_FILENAME = 'wifibox.stdout.log'
 local WIFIBOX_STDOUT_LOG_FILE = TMP_DIR .. '/' .. WIFIBOX_STDOUT_LOG_FILENAME
+local ROTATED_LOGS_DIRNAME = 'wifibox-rotated'
+local ROTATED_LOGS_DIR = TMP_DIR .. '/' .. ROTATED_LOGS_DIRNAME
 local MOD_ABBR = "AINF"
 
 local SYSLOG_FILENAME = 'syslog'
@@ -72,11 +74,28 @@ function M.logfiles(request, response)
 
 	--[[ create temporary files ]]--
 
-	-- copy wifibox API-script log
-	rv,sig,code = redirectedExecute('cp ' .. wifiboxLogFilePath .. ' ' .. LOG_COLLECT_DIR)
+	-- copy wifibox API-script (firmware) log
+	lfs.link(wifiboxLogFilePath, LOG_COLLECT_DIR .. '/' .. wifiboxLogFileName)
 
-	-- copy d3dapi script stdout/stderr log
-	rv,sig,code = redirectedExecute('cp ' .. WIFIBOX_STDOUT_LOG_FILE .. ' ' .. LOG_COLLECT_DIR)
+	-- copy d3dapi script stdout/stderr (fallback) log
+	lfs.link(WIFIBOX_STDOUT_LOG_FILE, LOG_COLLECT_DIR .. '/' .. WIFIBOX_STDOUT_LOG_FILENAME)
+
+	-- collect and copy print3d server logs
+	for file in lfs.dir(PRINT3D_BASEPATH) do
+		if file:find(PRINT3D_LOG_FILENAME_PREFIX) == 1 and file:find(PRINT3D_LOG_FILENAME_SUFFIX) ~= nil then
+			local srcLogFile = PRINT3D_BASEPATH .. '/' .. file
+			local tgtLogFile = LOG_COLLECT_DIR .. '/' .. file
+			lfs.link(srcLogFile, tgtLogFile)
+		end
+	end
+
+	-- copy rotated firmware and print3d logs
+	rv,msg = lfs.mkdir(LOG_COLLECT_DIR .. '/' .. ROTATED_LOGS_DIRNAME)
+	for file in lfs.dir(ROTATED_LOGS_DIR) do
+		local srcLogFile = ROTATED_LOGS_DIR .. '/' .. file
+		local tgtLogFile = LOG_COLLECT_DIR .. '/' .. ROTATED_LOGS_DIRNAME .. '/' .. file
+		lfs.link(srcLogFile, tgtLogFile)
+	end
 
 	-- capture syslog
 	rv,sig,code = os.execute('logread > ' .. LOG_COLLECT_DIR .. '/' .. SYSLOG_FILENAME)
@@ -97,6 +116,7 @@ function M.logfiles(request, response)
 	rv,sig,code = os.execute(USB_DIRTREE_COMMAND .. ' > ' .. LOG_COLLECT_DIR .. '/' .. USB_DIRTREE_FILENAME)
 
 	-- copy relevant openwrt configuration files
+	-- Note: we cannot link them because that would require the link to span over filesystems
 	rv,msg = lfs.mkdir(LOG_COLLECT_DIR .. '/config')
 	for _,v in pairs(UCI_CONFIG_FILES_TO_SAVE) do
 		local srcFile = '/etc/config/' .. v
@@ -104,19 +124,12 @@ function M.logfiles(request, response)
 		if v ~= 'wireless' then
 			rv,sig,code = redirectedExecute('cp ' .. srcFile .. ' ' .. tgtFile)
 		else
+			-- replace WiFi passwords with '...'
 			rv,sig,code = os.execute("sed \"s/option key '.*'/option key '...'/g\" " .. srcFile .. " > " .. tgtFile)
 		end
 	end
 
-	-- collect and copy print3d server logs
-	for file in lfs.dir(PRINT3D_BASEPATH) do
-		if file:find(PRINT3D_LOG_FILENAME_PREFIX) == 1 and file:find(PRINT3D_LOG_FILENAME_SUFFIX) ~= nil then
-			local srcLogFile = PRINT3D_BASEPATH .. '/' .. file
-			local tgtLogFile = LOG_COLLECT_DIR .. '/' .. file
-			rv,sig,code = redirectedExecute('cp ' .. srcLogFile .. ' ' .. tgtLogFile)
-			end
-		end
-
+	-- create tar.gz archive of the files/data we collected
 	rv,sig,code = redirectedExecute('tar czf ' .. LOG_COLLECT_ARCHIVE_FILE .. ' ' .. LOG_COLLECT_DIRNAME) --returns 0 success, 1 error
 
 
@@ -142,12 +155,17 @@ function M.logfiles(request, response)
 	rv,sig,code = redirectedExecute('rm ' .. LOG_COLLECT_DIR .. '/config/*')
 	rv,msg = lfs.rmdir(LOG_COLLECT_DIR .. '/config')
 
+	-- Note: this assumes the rotated logs directory does not contain subdirectories
+	rv,sig,code = redirectedExecute('rm ' .. LOG_COLLECT_DIR .. '/' .. ROTATED_LOGS_DIRNAME .. '/*')
+	rv,msg = lfs.rmdir(LOG_COLLECT_DIR .. '/' .. ROTATED_LOGS_DIRNAME)
+
 	rv,sig,code = redirectedExecute('rm ' .. LOG_COLLECT_DIR .. '/' .. USB_DIRTREE_FILENAME)
 	rv,sig,code = redirectedExecute('rm ' .. LOG_COLLECT_DIR .. '/' .. DISKFREE_FILENAME)
 	rv,sig,code = redirectedExecute('rm ' .. LOG_COLLECT_DIR .. '/' .. MOUNTS_FILENAME)
 	rv,sig,code = redirectedExecute('rm ' .. LOG_COLLECT_DIR .. '/' .. MEMINFO_FILENAME)
 	rv,sig,code = redirectedExecute('rm ' .. LOG_COLLECT_DIR .. '/' .. PROCESS_LIST_FILENAME)
 	rv,sig,code = redirectedExecute('rm ' .. LOG_COLLECT_DIR .. '/' .. SYSLOG_FILENAME)
+	rv,sig,code = redirectedExecute('rm ' .. LOG_COLLECT_DIR .. '/' .. WIFIBOX_STDOUT_LOG_FILENAME)
 	rv,sig,code = redirectedExecute('rm ' .. LOG_COLLECT_DIR .. '/' .. wifiboxLogFileName)
 
 	rv,msg = lfs.rmdir(LOG_COLLECT_DIR)

src/rest/api/api_printer.lua

@@ -120,7 +120,6 @@ end
 
 
 function M.heatup_POST(request, response)
-
 	if not accessManager.hasControl(request.remoteAddress) then
 		response:setFail("No control access")
 		return
@@ -145,8 +144,6 @@ function M.heatup_POST(request, response)
 end
 
 function M.stop_POST(request, response)
-	log:info(MOD_ABBR, "API:printer/stop")
-
 	if not accessManager.hasControl(request.remoteAddress) then
 		response:setFail("No control access")
 		return
@@ -183,8 +180,6 @@ end
 --accepts: seq_total(int) (total number of gcode chunks to be appended, must be given until clear() after given once, and stay the same)
 --returns: when the gcode buffer cannot accept the gcode, or the IPC transaction fails, a fail with a (formal, i.e., parseable) status argument will be returned
 function M.print_POST(request, response)
-	log:info(MOD_ABBR, "API:printer/print")
-
 	local controllerIP = accessManager.getController()
 	local hasControl = false
 	if controllerIP == "" then
@@ -194,7 +189,6 @@ function M.print_POST(request, response)
 		hasControl = true
 	end
 
-	log:info(MOD_ABBR, "  hasControl: "..utils.dump(hasControl))
 	if not hasControl then
 		response:setFail("No control access")
 		return
@@ -209,6 +203,8 @@ function M.print_POST(request, response)
 	local argSeqNumber = request:get("seq_number") or -1
 	local argSeqTotal = request:get("seq_total") or -1
 	local remoteHost = request:getRemoteHost()
+	
+	log:info(MOD_ABBR, "print chunk metadata: total_lines=" .. argTotalLines .. ", seq_number=" .. argSeqNumber .. ", seq_total=" .. argSeqTotal)
 
 	local printer,msg = printerUtils.createPrinterOrFail(argId, response)
 	if not printer or not printer:hasSocket() then return end

src/script/logrotate-wifibox.conf

@@ -0,0 +1,36 @@
+# Doodle3D logrotate configuration
+
+compress
+su
+create
+
+/tmp/wifibox.log /tmp/print3d-*.log {
+	rotate 2
+	size 1000k
+	missingok
+	olddir /tmp/wifibox-rotated
+
+	# as alternative to copytruncate (which *might* miss lines) we could implement
+	# SIGUSR1 in print3d and firmware to reopen their logs
+	copytruncate  # 'create' is ignored with copytruncate, as the file stays in place
+
+	prerotate
+		FIRST_N_LINES=500
+		PRINT3D_LOGHEAD_FILE=print3d-loghead
+		WIFIBOX_LOGHEAD_FILE=wifibox-loghead
+		ROTATED_FILES_PATH=/tmp/wifibox-rotated
+
+		#Note: by looking at creation date of the print3d socket and comparing against that
+		#      of print3d-loghead we could detect a server restart (e.g. due to reconnected printer)
+
+		echo "$1" | grep -q "print3d-.*\.log"
+		if [ $? -eq 0 -a ! -f "$ROTATED_FILES_PATH/$PRINT3D_LOGHEAD_FILE" ]; then
+			head -q -n$FIRST_N_LINES "$1" > "$ROTATED_FILES_PATH/$PRINT3D_LOGHEAD_FILE"
+		fi
+
+		echo "$1" | grep -q "wifibox.log"
+		if [ $? -eq 0 -a ! -f "$ROTATED_FILES_PATH/$WIFIBOX_LOGHEAD_FILE" ]; then
+			head -q -n$FIRST_N_LINES "$1" > "$ROTATED_FILES_PATH/$WIFIBOX_LOGHEAD_FILE"
+		fi
+	endscript
+}

src/script/wifibox_init

@@ -17,9 +17,11 @@ boot() {
 	$LOGGER "Invoking Doodle3D WiFi box network auto-initialization..."
 	/usr/share/lua/wifibox/script/d3dapi autowifi
 	
-  $LOGGER "Start signing in..."
-  /usr/share/lua/wifibox/script/signin.sh > /dev/null 2> /dev/null &
+	$LOGGER "Start signing in..."
+	/usr/share/lua/wifibox/script/signin.sh > /dev/null 2> /dev/null &
 	
+	mkdir -p /var/lib  # required by logrotate for logrotate.status
+	mkdir -p /tmp/wifibox-rotated  # this is where rotated wifibox logs are placed
 }
 
 start() {