#!/bin/sh
# run_awstats v1.3.9  (c) 3.6.2004 by Andreas Ley  (u) 24.7.2025
# Build awstats overview

# Debugging:
# To run this script manually, call run_awstats as the user of the corresponding apache and with the FQDNs to process as arguments
# E.g.: owner -u <UID>:<GID> run_awstats <FQDN>
# Logs of runs of this script are stored as /var/www/log/run_awstats/<UID>/<PID>.*
# If a file .debug exists in /home/ws/<USER>/<FQDN>/htdocs/awstats, copied logs aren't discared but stored together with the awstats files in /var/www/<FQDN>/run/awstats.<DATE>
# If this file isn't empty, the first line is given as -debug= option to awstats

umask 022

# DEBUG
log="/var/www/log/run_awstats/$(id -u)"
find "${log}" -mtime +7 -delete
test -w "${log}" && exec >"${log}/$$.out" 2>"${log}/$$.err" && set -x

# http://awstats.sourceforge.net/docs/

admin="apache@scc.kit.edu"
cc="webmaster@kit.edu"
cc="webmaster@kit.edu,Ulrich.Betzler@kit.edu"

subpages="alldomains allhosts lasthosts unknownip alllogins lastlogins allrobots lastrobots urldetail urlentry urlexit browserdetail osdetail unknownbrowser unknownos refererse refererpages keyphrases keywords errors404 downloads"
subpages_urlfilter="urldetail urlentry urlexit"

awstats="/usr/lib/cgi-bin/awstats.pl"
conf_orig="/etc/awstats"
root="/var/www"

# For the transit, collect from all clusters
#servers="${root}/conf/servers"
#servers="/etc/cluster/nodes"
servers="/etc/cluster/nodes.all"

timeout=10

PATH=/usr/sbin:/usr/bin:/sbin:/bin
export PATH

usage()
{
	echo "Usage: $(basename $0) [-f] [-o] [-r|-R] vhost [...]" >&2
	echo "-f  force running on this host" >&2
	echo "-o  output only, don't gather log files or update history" >&2
	echo "-r  also rebuild all monthly and yearly statistics" >&2
	echo "-R  also rebuild all missing monthly and yearly statistics" >&2
	exit 1
}

set -- $(getopt hxvd:forR $*) || usage

trace=false
verbose=false
force=false
update=true
rebuild=false
rebuild_if_missing=false
while :; do
	case $1 in
		-h)	sed '1d;s/^# *//;/^$/q' $0; usage;;
		-x)	set -x; trace=true; shift;;
		-v)	verbose=true; shift;;
		-d)	debug="$2"; shift 2;;
		-f)	force=true; shift;;
		-o)	update=false; shift;;
		-r)	rebuild=true; shift;;
		-R)	rebuild_if_missing=true; shift;;
		--)	shift; break;;
	esac
done

test $# -eq 0 && usage

mkstats()
{
	${trace} && set -x
# DEBUG
set -x

	opts="$1"
	path="$2"

	#ls -1af | egrep '^\.(awstats\..*|index)\.html\.[0-9]*$' | xargs rm -f

	for subpage in ${subpages}; do
		page="${path}awstats.${hostname}.${subpage}.html"
		tmp="${path}.awstats.${hostname}.${subpage}.html.$$"
		${awstats} -config="${hostname}" -urlfilterex='^/(~|%7[eE])' ${opts} -output="${subpage}" -staticlinks > "${tmp}" && chmod 444 "${tmp}" && mv -f "${tmp}" "${page}"
	done

	page="${path}awstats.${hostname}.urldetail.user.html"
	tmp="${path}.awstats.${hostname}.urldetail.user.html.$$"
	test -f "${page}" && \
		${awstats} -config="${hostname}" -urlfilter='^/(~|%7[eE])' ${opts} -output="urldetail" -staticlinks > "${tmp}" && chmod 444 "${tmp}" && mv -f "${tmp}" "${page}"

	if test -s "${vhost}/conf/awstats.urlfilter"; then
		for subpage in ${subpages_urlfilter}; do
			base="awstats.${hostname}.${subpage}.html"
			page="${path}${base}"
			tmp="${path}.${base}.$$"
			# Patch urlfilter links to global statistics subpage
			sed -n '1,/Fenster schlie&szlig/p' "${page}" >"${tmp}"
			echo "</table>" >>"${tmp}"
			echo "<br />" >>"${tmp}"
			echo >>"${tmp}"
			echo "<table>" >>"${tmp}"
			echo "<tr><td class=\"awsm\" width=\"150\" valign=\"top\"><b>Filter:</b></td>" >>"${tmp}"
			echo -n "<td class=\"awsm\">" >>"${tmp}"
			while read urlfilter; do
				suffix="$(echo -n "${urlfilter}" | tr -c '[a-zA-Z0-9]' '.')"
				fbase="awstats.${hostname}.${subpage}${suffix}.html"
				fpage="${path}${fbase}"
				ftmp="${path}.${fbase}.$$"
				${awstats} -config="${hostname}" -urlfilter="${urlfilter}" -urlfilterex='^/(~|%7[eE])' ${opts} -output="${subpage}" -staticlinks > "${ftmp}" && chmod 444 "${ftmp}" && mv -f "${ftmp}" "${fpage}"
				echo -n "<a href=\"${fbase}\">${urlfilter}</a> &nbsp; " >>"${tmp}"
			done <"${vhost}/conf/awstats.urlfilter"
			echo "</td></tr>" >>"${tmp}"
			sed '1,/Fenster schlie&szlig/d' "${page}" >>"${tmp}"
			chmod 444 "${tmp}"
			mv -f "${tmp}" "${page}"
		done
	fi

	page="${path}index.html"
	tmp="${path}.index.html.$$"
	${awstats} -config="${hostname}" -urlfilterex='^/(~|%7[eE])' ${opts} -output -staticlinks > "${tmp}" && chmod 444 "${tmp}" && mv -f "${tmp}" "${page}"
}

thisyear="$(date +%Y)"
thismonth="$(date +%m)"

for hostname; do
	conf="${conf_orig}"
	if test ! -s "${conf}/awstats.${hostname}.conf"; then
		echo "No config: ${conf}/awstats.${hostname}.conf" >&2
		continue
	fi

	dirdata="$(shstrings -v -f "${conf}/awstats.${hostname}.conf" DirData)"
	awstatsdir="$(dirname "${dirdata}")"
	# It's legal to remove the awstats directory, just ignore it
	test -d "${awstatsdir}" || continue
	# But if this fails, something is weird
	if ! cd "${awstatsdir}"; then
		echo "Can't chdir to ${awstatsdir}" >&2
		continue
	fi
	test -f .debug && debug="$(head -1 .debug)"
	# Just in case...
	if ! mkdir -p "${dirdata}"; then
		echo "Can't mkdir ${dirdata}" >&2
		continue
	fi
	test -L doc || ln -s /usr/share/doc/awstats/html doc

	# Remove ephemeral files from earlier broken runs
	find 'data' -type f -name "awstats*.${hostname}.tmp*" -ctime +0 -delete

	vhost="${root}/${hostname}"

	# Include a local awstats config
	if test -s "${vhost}/conf/awstats.local"; then
# Does not work due to root-owned ${conf}/awstats.${hostname}.conf
#		if ! grep -q "^Include \"${vhost}/conf/awstats.local\"" "${conf}/awstats.${hostname}.conf"; then
#			echo >>"${conf}/awstats.${hostname}.conf"
#			echo "Include \"${vhost}/conf/awstats.local\"" >>"${conf}/awstats.${hostname}.conf"
#		fi
		echo "Include \"${vhost}/conf/awstats.local\"" | cat "${conf}/awstats.${hostname}.conf" - >"${vhost}/run/awstats.${hostname}.conf"
		conf="${vhost}/run"
	fi

# There may be a leftover...
rm -f "${vhost}/run/awstats.pl"

	if ${update}; then

		# Copy log files from remote nodes or filter local log files
		dir="${vhost}/run/awstats"
		rm -rf "${dir}"
		mkdir -p "${dir}"

		# Remember when we last started a logfile collection run
		time="$(date '+%s')"
		timestamp="$(date '+%d%m%y %H%M%S')"
		if test -s "${dirdata}/.timestamp"; then
			start="-s $(head -1 "${dirdata}/.timestamp")"
			get="$(stat -c %Y "${dirdata}/.timestamp")"
		else
			# If there is no timestamp for the last logfile collection, we take what we can get
			start="-w"
			get=0
		fi

		# If we have to merge files within the cluster, first copy them to the local node
		if grep "^LogFile=.*logresolvemerge" "${conf}/awstats.${hostname}.conf" >/dev/null; then
			# Determine local and available nodes to select from
			if test -z "${localnode}"; then
				localnode="$(hostname --fqdn)"
				nodes="$(/bin/mktemp)"
				trap "rm -f \"${nodes}\"" 0 1 2 3 15
				awk '/^[^#]/{print$1}' "${servers}" | fping -a -r1 >"${nodes}" 2>/dev/null
			fi

			# Don't run but on the selected node
			#${force} || test "${localnode}" = "$(select_node "${hostname}" <"${nodes}")" || continue
			${force} || test "${localnode}" = "$(egrep -v '(test|legacy)' "${nodes}" | select_node "${hostname}")" || continue
			while read node; do
				if test "${node}" = "${localnode}"; then
					#myaccesslog -V "${hostname}" ${start} >"${dir}/${node}"
					case ${node} in
						web*)
							getlogs ${hostname} ${get} >"${dir}/${node}"
							;;
						*)	ln -s "${vhost}/log/access.log.1" "${dir}/${node}"
							;;
					esac
				else
					# When a password has expired, ssh1 asks for a
					# new password - this can be avoided with -n,
					# but there is no keyword for -n, and scp1 can
					# only give keywords to ssh1. We're doomed...
					#scp1 -p -B -C "${node}:${vhost}/logs/access.log.1" "${PWD}/logs/${node}"
					# Fortunatly, scp2 ignores expired passwords
					# in batch mode completly :)
					#scp2 -p -B -o "Compression=yes" -q -Q "${node}:${vhost}/logs/access.log.1" "${PWD}/logs/${node}"
					#ssh1 -q -n -a -k -x -o "BatchMode=yes" -o "Compression=yes" "${node}" "exec /usr/bin/myaccesslog -V ${hostname} ${start}" >"${PWD}/logs/${node}"
					# It's only marginal, but Compression=on reduces system load and total run time
					case ${node} in
#						legacy*)
#							/usr/bin/ssh -q -n -a -k -x -o "BatchMode=yes" -o "Compression=yes" -o "ConnectTimeout=${timeout}" "${node}" "exec /usr/bin/myaccesslog -V ${hostname} ${start}" >"${dir}/${node}"
#							;;
						web*|legacy*)
							/usr/bin/ssh -q -n -a -k -x -o "BatchMode=yes" -o "Compression=yes" -o "ConnectTimeout=${timeout}" "${node}" "exec /usr/sbin/getlogs ${hostname} ${get}" >"${dir}/${node}"
							;;
						*)
							q='-q'
							test -f .debug && q='-v'
							scp -p -B -C ${q} -o "ConnectTimeout=${timeout}" "${node}:${vhost}/log/access.log.1" "${dir}/${node}"
							test -f .debug && ls -ld "${dir}" >&2
							test -f .debug && ls -l "${dir}/${node}" >&2
							;;
					esac
				fi
			done <"${nodes}"
		else
			#myaccesslog -V "${hostname}" ${start} >"${dir}/access.log"
			ln -s "${vhost}/log/access.log.1" "${dir}/access.log"
		fi

		# Update awstats database with logfile data
		out="${vhost}/run/awstats.out.$$"
		if ! ${awstats} -configdir="${conf}" -config="${hostname}" -update ${debug:+-debug="${debug}"} >"${out}"; then
			#mailaddr="$(id -nu)@sysmail.kit.edu"
			mailaddr="${admin}"
			(echo "From: ${admin}"
			echo "To: ${mailaddr}"
			#test -n "${cc}" && echo "Cc: ${cc}"
			#echo "Bcc: ${admin}"
			echo "Subject: awstats for ${hostname}${localnode:+ on ${localnode}} failed"
			echo
			see=
			test -s "${out}" && see=" See the following awstats output for further information."
			# Can't use the echo builtin since it unavoidably interprets the backslashes in the CIFS share path
			/bin/echo "There has been a problem while updating the advanced web statistics. This could be due to exceeded quota for your account (you can see your quota usage in \$HOME/Quota.txt when logged in to the webserver or $(sharepath -c -U $(id -u))\\Quota.txt on windows) or a permissions problem (for awstats to be able to write you should own the ${awstatsdir} and ${dirdata} directories and the files therein).${see} If you can't fix the problem yourself, please contact ${admin}${cc:+ or ${cc}}." | lbreak -c 72
			test -n "${see}" && echo
			# No envelope-from to avoid NDRs
			#cat "${out}") | /usr/lib/sendmail -i -f '' -t
			#cat "${out}") | /usr/lib/sendmail -i -t
			cat "${out}"
			echo
			echo "For SCC-internal debugging, here are the hard mounts:"
			echo
			mount | grep /home/hard/) | /usr/lib/sendmail -i -t
			continue
		fi
		rm -f "${out}"
		# Remember where to start the next logfile collection run
		test -n "${timestamp}" && \
			echo "${timestamp}" >"${dirdata}/.timestamp" && \
			touch -d "@${time}" "${dirdata}/.timestamp"

		# Save logs and history for debugging
		if test -f .debug; then
			cp -pr data awstats.${hostname}.*.html "${dir}"
			mv -f "${dir}" "${dir}.$(date '+%Y-%m-%dT%H:%M:%S')"
		fi
		# Discard log backups older than a week
		find "${vhost}/run" -mindepth 1 -maxdepth 1 -type d -name 'awstats.*' -ctime +7 -execdir rm -rf '{}' +
		# Discard copied logs
		rm -rf "${dir}"
	fi

	# Build main statistics
	mkstats "-configdir=${conf}"

	# Build archive statistics
	ls -1 data | sed -n "s!^awstats\\([0-9][0-9]\\)\\([0-9][0-9][0-9][0-9]\\)\\.${hostname}\\.txt\$!\\1 \\2!p" | while read month year; do
		if ${rebuild} || ${rebuild_if_missing} && test ! -s "${year}/${month}/index.html" || test \( "${year}" -ne "${thisyear}" -o "${month}" -ne "${thismonth}" \) -a ! -s "${year}/${month}/index.html"; then
			mkdir -p "${year}/${month}"

			# Build monthly statistics
			cat "${conf}/awstats.${hostname}.conf" >"awstats.${hostname}.conf"
			echo "ShowMonthStats=0" >>"awstats.${hostname}.conf"
			mkstats "-configdir=. -year=${year} -month=${month}" "${year}/${month}/"

			# Build yearly statistics
			# (not inefficient, since we normally only build one monthly statistic every month, and the yearly statistics should be updated every month)
			cat "${conf}/awstats.${hostname}.conf" >"awstats.${hostname}.conf"
			echo "ShowDaysOfMonthStats=0" >>"awstats.${hostname}.conf"
			mkstats "-configdir=. -year=${year} -month=all" "${year}/"

			rm -f "awstats.${hostname}.conf"
		fi
	done

	# Patch archive links to main statistics index
#	cp -p "index.html" "index-orig.html"
	#sed -n '1,/Nicht gefundene Seiten/p' "index.html" >".index.html.$$"
	sed -n '1,/#misc/p' "index.html" >".index.html.$$"

	if test -s ../linkchecker/linkchecker-out.html; then
		echo " &nbsp; </td></tr>"
		echo "<tr><td class=\"awsm\" valign=\"top\"><b>Linkchecker:</b></td>"
		echo "<td class=\"awsm\"><a href=\"/linkchecker/linkchecker-out.html\">Fehlerhafte Seiten</a>"
	fi >>".index.html.$$"

	header="<b>Archiv:</b>"
	ls -1 | grep '^[0-9][0-9][0-9][0-9]' | sort -n | while read year; do
		echo " &nbsp; </td></tr>"
		echo "<tr><td class=\"awsm\" valign=\"top\">${header}</td>"
		header=""
		echo "<td class=\"awsm\"><a href=\"${year}/\">${year}:</a>"
		ls -1 "${year}" | grep '^[0-9][0-9]' | sort -n | while read month; do
			echo " &nbsp; <a href=\"${year}/${month}/\">$(sed -n "s!^message$(expr 59 + ${month})=!!p" /usr/share/awstats/lang/awstats-de.txt)</a>"
		done
	done >>".index.html.$$"
	#sed '1,/Nicht gefundene Seiten/d' "index.html" >>".index.html.$$"
	sed '1,/#misc/d' "index.html" >>".index.html.$$"
	chmod 444 ".index.html.$$"
	mv -f ".index.html.$$" "index.html"
done

exit 0
