xref: /netbsd-src/external/public-domain/tz/tzdata2netbsd (revision fb5eed702691094bd687fbf1ded189c87457cd35) 
1# $NetBSD: tzdata2netbsd,v 1.12 2018/03/24 01:54:48 kre Exp $
2
3# For use by NetBSD developers when updating to new versions of tzdata.
4#
5# 0. Be in an up-to-date checkout of src/external/public-domain/tz
6#    from NetBSD-current.
7# 1. Make sure that you have Paul Eggert's 4K RSA public key in your
8#    keyring (62AA7E34, eggert@cs.ucla.edu)  It is not required that it be trusted.
9# 2. Run this script.  You will be prompted for confirmation before
10#    anything major (such as a cvs operation).  The tz versions can be
11#    specified as args (new version first, and the previous second) if
12#    needed to override the calculated values
13# 3. If something fails, abort the script and fix it.
14# 4. Re-run this script until you are happy.  It's designed to
15#    be re-run over and over, and later runs will try not to
16#    redo non-trivial work done by earlier runs.
17#
18
19VERS_PATTERN='2[0-9][0-9][0-9][a-z]'
20# This needs to be updated twice every millennium to allow for the
21# new millenium's years.
22# First in the late xx90's sometime, allow the new one by changing the leading
23# digit from a specific value to the class containing the current and
24# following values (eg: in 2098 or so, change '2' to be '[23]').
25# Tnen in the following early xx00's sometime, delete the class, and insert
26# leave only the current value as valid (eg: in 3001 or 3002,
27# change '[23]' to be just '3'
28# Doing it this way helps guard against invalid specifications.
29# We could automate this, but it is (IMO) not worth the cost, to avoid a
30# twice a millenium edit requirement.
31# A more significant (and harder) change will be needed in the late 9990's
32# If this script lasts until then, send me a postcard, I'll be waiting for it!
33# Things get easier again after that until the late 99990's (etc.)
34
35# Note the pattern is used on both the old and new version specifiers,
36# so it must be able to cope with the shift from one form (eg 2999g)
37# to the new one (eg: 3000a) without failing (or the code that uses it
38# below needs to be updated).
39
40# Also note that the option of having a second alpha (1997aa or something)
41# to handle years with much activity is handled below, the pattern does not
42# need to match those.
43# If that convention changes (as of date of writing, it has never been
44# exercised) then code changes below will be required.
45
46DIST_HOST=ftp.iana.org
47DIST_PATH=tz
48DIST_FILES=releases
49
50EDITOR=${EDITOR:-vi}
51WORK_PFX=$(pwd)/update-work || fail "Cannot obtain PWD"
52UPDATE_FROM=${WORK_PFX}/updating.from.version
53
54usage()
55{
56	printf >&2 '%s\n'						\
57		"Usage: $0 [new-version-id [old-version-id]]"		\
58		"     where a version-id is of the form YYYYx (eg: 2018c)" \
59		"     or '' for new-version-id (to specify only the old)"
60	exit 2
61}
62
63fail()
64{
65	local IFS=' '
66
67	printf >&2 '%s\n'	"Error detected:" "   $*" "Aborting."
68	exit 1
69}
70
71valid_vers()
72{
73	case "$2" in
74	${VERS_PATTERN} | ${VERS_PATTERN}[a-z] )
75		;;
76	*)	printf >&2 '%s: %s\n' \
77		    "Bad form for $1 version specifier '$2'" \
78		    "should (usually) be 'YYYYx'"
79		return 1
80		;;
81	esac
82	return 0
83}
84
85get_curvers()
86{
87	local LF=''
88	local LIST=iana-listing
89	local SED_SCRIPT='
90		/tzdata-latest.*-> /{
91			s/^.*-> //
92			s/\..*$//
93			s;^releases/tzdata;;p
94			q
95		}
96		d'
97
98	test -d "${WORK_PFX}" &&
99		test -s "${WORK_PFX}/${LIST}" &&
100		test "${WORK_PFX}/${LIST}" -nt dist/CVS &&
101		LF=$(find "${WORK_PFX}" -name "${LIST}" -mtime -1 -print) &&
102		test -n "${LF}" &&
103		NEWVER=$(sed -n < "${LF}" "${SED_SCRIPT}") &&
104		valid_vers new "${NEWVER}"				||
105
106	ftp >/dev/null 2>&1 -ia "${DIST_HOST}" <<- EOF &&
107					dir ${DIST_PATH} ${WORK_PFX}/${LIST}
108					quit
109					EOF
110		test -s "${WORK_PFX}/${LIST}" &&
111		NEWVER=$(sed -n < "${WORK_PFX}/${LIST}" "${SED_SCRIPT}") &&
112		valid_vers new "${NEWVER}"				||
113
114	{
115		rm -f "${WORK_PFX}/${LIST}"
116		fail "Cannot fetch current tzdata version from ${DIST_HOST}"
117	}
118
119	printf '%s\n' "Updating from ${OLDVER} to ${NEWVER}"
120}
121
122argparse()
123{
124	local OVF
125
126	if OVF=$(find "${WORK_PFX}" -name "${UPDATE_FROM##*/}" -mtime +2 -print)
127	then
128		# delete anything old
129		test -n "${OVF}" && rm -f "${OVF}"
130	fi
131
132	case "$#" in
133	0|1)
134		# once we have obtained OLDVER once, never guess it again.
135		test -f "${UPDATE_FROM}" && OLDVER=$(cat "${UPDATE_FROM}") ||
136
137		OLDVER=$(cat dist/version) || {
138			printf >&2 '%s\n'  \
139			    "Cannot determine current installed version"  \
140			    "Specify it on the command line."  \
141			    ""
142			usage
143		}
144
145		valid_vers old "${OLDVER}" ||
146			fail "Calculated bad OLDVER, give as 2nd arg"
147		;;
148
149	2)	valid_vers old "$2" && OLDVER="$2" || usage
150		;;
151
152	*)	usage
153		;;
154	esac
155
156	case "$#:$1" in
157	0: | 1: | 2: )
158		;;
159	1:?*|2:?*)
160		valid_vers new "$1" && NEWVER="$1" || usage
161		;;
162	*)	usage
163		;;
164	esac
165
166	test -z "${NEWVER}" && get_curvers
167
168	test "${NEWVER}" = "${OLDVER}" && {
169		printf '%s\n' \
170		     "New and old versions both ${NEWVER}: nothing to do"
171		exit 0
172
173	}
174
175	printf '%s\n' "${OLDVER}" > "${UPDATE_FROM}" ||
176	    fail "Unable to preserve old version ${OLDVER} in ${UPDATE_FROM}"
177
178	test "${#NEWVER}" -gt "${#OLDVER}" ||
179		test "${NEWVER}" '>' "${OLDVER}" ||
180		{
181			local reply
182
183			printf '%s\n' \
184			    "Update would revert ${OLDVER} to ${NEWVER}"
185			read -p "Is reversion intended? " reply
186			case "${reply}" in
187			[Yy]*)	;;
188			*)	printf '%s\n' OK. Aborted.
189				rm -f "${UPDATE_FROM}"
190				exit 1
191				;;
192			esac
193		}
194
195	return 0
196}
197
198setup_versions()
199{
200	# Uppercase variants of OLDVER and NEWVER
201	OLDVER_UC="$( echo "${OLDVER}" | tr '[a-z]' '[A-Z]' )"
202	NEWVER_UC="$( echo "${NEWVER}" | tr '[a-z]' '[A-Z]' )"
203
204	# Tags for use with version control systems
205	CVSOLDTAG="TZDATA${OLDVER_UC}"
206	CVSNEWTAG="TZDATA${NEWVER_UC}"
207	CVSBRANCHTAG="TZDATA"
208	GITHUBTAG="${NEWVER}"
209
210	# URLs for fetching distribution files, etc.
211	DISTURL="ftp://${DIST_HOST}/${DIST_PATH}/${DIST_FILES}"
212	DISTURL="${DISTURL}/tzdata${NEWVER}.tar.gz"
213	SIGURL="${DISTURL}.asc"
214	NEWSURL="https://github.com/eggert/tz/raw/${GITHUBTAG}/NEWS"
215
216	# Directories
217	REPODIR="src/external/public-domain/tz/dist"
218				# relative to the NetBSD CVS repo
219	TZDISTDIR="$(pwd)/dist" # should be .../external/public-domain/tz/dist
220	WORKDIR="${WORK_PFX}/${NEWVER}"
221	EXTRACTDIR="${WORKDIR}/extract"
222
223	# Files in the work directory
224	DISTFILE="${WORKDIR}/${DISTURL##*/}"
225	SIGFILE="${DISTFILE}.asc"
226	PGPVERIFYLOG="${WORKDIR}/pgpverify.log"
227	NEWSFILE="${WORKDIR}/NEWS"
228	NEWSTRIMFILE="${WORKDIR}/NEWS.trimmed"
229	IMPORTMSGFILE="${WORKDIR}/import.msg"
230	IMPORTDONEFILE="${WORKDIR}/import.done"
231	MERGSMSGFILE="${WORKDIR}/merge.msg"
232	MERGEDONEFILE="${WORKDIR}/merge.done"
233	COMMITMERGEDONEFILE="${WORKDIR}/commitmerge.done"
234
235	printf '%s\n' "${CVSOLDTAG}"  > "${WORK_PFX}/updating_from"
236}
237
238DOIT()
239{
240	local really_do_it=false
241	local reply
242
243	echo "In directory $(pwd)"
244	echo "ABOUT TO DO:" "$(shell_quote "$@")"
245	read -p "Really do it? [yes/no/quit] " reply
246	case "${reply}" in
247	[yY]*)	really_do_it=true ;;
248	[nN]*)	really_do_it=false ;;
249	[qQ]*)
250		echo "Aborting"
251		return 1
252		;;
253	*)	echo "Huh?"
254		return 1
255		;;
256	esac
257	if $really_do_it; then
258		echo "REALLY DOING IT NOW..."
259		"$@"
260	else
261		echo "NOT REALLY DOING THE ABOVE COMMAND"
262	fi
263}
264
265# Quote args to make them safe in the shell.
266# Usage: quotedlist="$(shell_quote args...)"
267#
268# After building up a quoted list, use it by evaling it inside
269# double quotes, like this:
270#    eval "set -- $quotedlist"
271# or like this:
272#    eval "\$command $quotedlist \$filename"
273#
274shell_quote()
275(
276	local result=''
277	local arg qarg
278	LC_COLLATE=C ; export LC_COLLATE # so [a-zA-Z0-9] works in ASCII
279	for arg in "$@" ; do
280		case "${arg}" in
281		'')
282			qarg="''"
283			;;
284		*[!-./a-zA-Z0-9]*)
285			# Convert each embedded ' to '\'',
286			# then insert ' at the beginning of the first line,
287			# and append ' at the end of the last line.
288			# Finally, elide unnecessary '' pairs at the
289			# beginning and end of the result and as part of
290			# '\'''\'' sequences that result from multiple
291			# adjacent quotes in he input.
292			qarg="$(printf '%s\n' "$arg" | \
293			    ${SED:-sed} -e "s/'/'\\\\''/g" \
294				-e "1s/^/'/" -e "\$s/\$/'/" \
295				-e "1s/^''//" -e "\$s/''\$//" \
296				-e "s/'''/'/g"
297				)"
298			;;
299		*)
300			# Arg is not the empty string, and does not contain
301			# any unsafe characters.  Leave it unchanged for
302			# readability.
303			qarg="${arg}"
304			;;
305		esac
306		result="${result}${result:+ }${qarg}"
307	done
308	printf '%s\n' "$result"
309)
310
311validate_pwd()
312{
313	local P="$(pwd)" || return 1
314
315	test -d "${P}" &&
316	    test -d "${P}/CVS" &&
317	    test -d "${P}/dist" &&
318	    test -f "${P}/dist/zone.tab" &&
319	    test -f "${P}/tzdata2netbsd" || {
320		printf >&2 '%s\n' "Please change to the correct directory"
321		return 1
322	}
323
324	test -f "${P}/CVS/Tag" && {
325
326		# Here (for local use only) if needed for private branch work
327		# insert tests for the value of $(cat "${P}/CVS/Tag") and
328		# allow your private branch tag to pass. Eg:
329
330		#	case "$(cat "${P}/CVS/Tag")" in
331		#	my-branch-name)	return 0;;
332		#	esac
333
334		# Do not commit a version of this script modified that way,
335		# (not even on the private branch) - keep it as a local
336		# modified file.  (This script will not commit it.)
337
338		printf >&2 '%s\n' \
339		    "This script should be run in a checkout of HEAD only"
340		return 1
341	}
342
343	return 0
344}
345
346findcvsroot()
347{
348	[ -n "${CVSROOT}" ] && return 0
349	CVSROOT="$( cat ./CVS/Root )"
350	[ -n "${CVSROOT}" ] && return 0
351	echo >&2 "Failed to set CVSROOT value"
352	return 1
353}
354
355mkworkpfx()
356{
357	mkdir -p "${WORK_PFX}" || fail "Unable to make missing ${WORK_PFX}"
358}
359mkworkdir()
360{
361	mkdir -p "${WORKDIR}" || fail "Unable to make missing ${WORKDIR}"
362}
363
364fetch()
365{
366	[ -f "${DISTFILE}" ] || ftp -o "${DISTFILE}" "${DISTURL}" ||
367		fail "fetch of ${DISTFILE} failed"
368	[ -f "${SIGFILE}" ] || ftp -o "${SIGFILE}" "${SIGURL}" ||
369		fail "fetch of ${SIGFILE} failed"
370	[ -f "${NEWSFILE}" ] || ftp -o "${NEWSFILE}" "${NEWSURL}" ||
371		fail "fetch of ${NEWSFILE} failed"
372}
373
374checksig()
375{
376	{ gpg --verify "${SIGFILE}" "${DISTFILE}"
377	  echo gpg exit status $?
378	} 2>&1 | tee "${PGPVERIFYLOG}"
379
380	# The output should contain lines that match all the following regexps
381	#
382	while read line; do
383		if ! grep -E -q -e "^${line}\$" "${PGPVERIFYLOG}"; then
384			echo >&2 "Failed to verify signature: ${line}"
385			return 1
386		fi
387	done <<'EOF'
388gpg: Signature made .* using RSA key ID (62AA7E34|44AD418C)
389gpg: Good signature from "Paul Eggert <eggert@cs.ucla.edu>"
390gpg exit status 0
391EOF
392}
393
394extract()
395{
396	[ -f "${EXTRACTDIR}/zone.tab" ] && return
397	mkdir -p "${EXTRACTDIR}"
398	tar -z -xf "${DISTFILE}" -C "${EXTRACTDIR}"
399}
400
401addnews()
402{
403	[ -f "${EXTRACTDIR}/NEWS" ] && return
404	cp -p "${NEWSFILE}" "${EXTRACTDIR}"/NEWS
405}
406
407# Find the relevant part of the NEWS file for all releases between
408# OLDVER and NEWVER, and save them to NEWSTRIMFILE.
409#
410trimnews()
411{
412	[ -s "${NEWSTRIMFILE}" ] && return
413	awk -v oldver="${OLDVER}" -v newver="${NEWVER}" \
414	    '
415		BEGIN {inrange = 0}
416		/^Release [0-9]+[a-z]+ - .*/ {
417			# "Release <version> - <date>"
418			# Note: must handle transition from 2018z to 2018aa
419			# Assumptions: OLDVER and NEWVER have been sanitized,
420			# and format of NEWS file does not alter (and
421			# contains valid data)
422			inrange = ((length($2) > length(oldver) || \
423					$2 > oldver) && \
424				(length($2) < newver || $2 <= newver))
425		}
426		// { if (inrange) print; }
427	    ' \
428		<"${NEWSFILE}" >"${NEWSTRIMFILE}"
429	echo "tzdata-${NEWVER}" > ${TZDISTDIR}/TZDATA_VERSION
430}
431
432# Create IMPORTMSGFILE from NEWSTRIMFILE, by ignoring some sections,
433# keeping only the first sentence from paragraphs in other sections,
434# and changing the format.
435#
436# The result should be edited by hand before performing a cvs commit.
437# A message to that effect is inserted at the beginning of the file.
438#
439mkimportmsg()
440{
441	[ -s "${IMPORTMSGFILE}" ] && return
442	{ cat <<EOF
443EDIT ME: Edit this file and then delete the lines marked "EDIT ME".
444EDIT ME: This file will be used as a log message for the "cvs commit" that
445EDIT ME: imports tzdata${NEWVER}.  The initial contents of this file were
446EDIT ME: generated from ${NEWSFILE}.
447EDIT ME:
448EOF
449	awk -v oldver="${OLDVER}" -v newver="${NEWVER}" \
450	    -v disturl="${DISTURL}" -v newsurl="${NEWSURL}" \
451	    '
452		BEGIN {
453			bullet = "  * ";
454			indent = "    ";
455			blankline = 0;
456			goodsection = 0;
457			havesentence = 0;
458			print "Import tzdata"newver" from "disturl;
459			#print "and NEWS file from "newsurl;
460		}
461		/^Release/ {
462			# "Release <version> - <date>"
463			ver = $2;
464			date = gensub(".* - ", "", 1, $0);
465			print "";
466			print "Summary of changes in tzdata"ver \
467				" ("date"):";
468		}
469		/^$/ { blankline = 1; havesentence = 0; }
470		/^  Changes affecting/ { goodsection = 0; }
471		/^  Changes affecting.*time/ { goodsection = 1; }
472		/^  Changes affecting.*data/ { goodsection = 1; }
473		/^  Changes affecting.*documentation/ || \
474		/^  Changes affecting.*commentary/ {
475			t = gensub("^ *", "", 1, $0);
476			t = gensub("\\.*$", ".", 1, t);
477			print bullet t;
478			goodsection = 0;
479		}
480		/^    .*/ && goodsection {
481			# In a paragraph in a "good" section.
482			# Ignore leading spaces, and ignore anything
483			# after the first sentence.
484			# First line of paragraph gets a bullet.
485			t = gensub("^ *", "", 1, $0);
486			t = gensub("\\. .*", ".", 1, t);
487			if (blankline) print bullet t;
488			else if (! havesentence) print indent t;
489			havesentence = (havesentence || (t ~ "\\.$"));
490		}
491		/./ { blankline = 0; }
492	    ' \
493		<"${NEWSTRIMFILE}"
494	} >"${IMPORTMSGFILE}"
495}
496
497editimportmsg()
498{
499	if [ -s "${IMPORTMSGFILE}" ] \
500	&& ! grep -q '^EDIT' "${IMPORTMSGFILE}"
501	then
502		return 0 # file has already been edited
503	fi
504	# Pass both IMPORTMSGFILE and NEWSFILE to the editor, so that the
505	# user can easily consult NEWSFILE while editing IMPORTMSGFILE.
506	${EDITOR} "${IMPORTMSGFILE}" "${NEWSFILE}"
507}
508
509cvsimport()
510{
511	if [ -e "${IMPORTDONEFILE}" ]; then
512		cat >&2 <<EOF
513The CVS import has already been performed.
514EOF
515		return 0
516	fi
517	if ! [ -s "${IMPORTMSGFILE}" ] \
518	|| grep -q '^EDIT' "${IMPORTMSGFILE}"
519	then
520		cat >&2 <<EOF
521The message file ${IMPORTMSGFILE}
522has not been properly edited.
523Not performing cvs import.
524EOF
525		return 1
526	fi
527	( cd "${EXTRACTDIR}" &&
528	  DOIT cvs -d "${CVSROOT}" import -m "$(cat "${IMPORTMSGFILE}")" \
529		"${REPODIR}" "${CVSBRANCHTAG}" "${CVSNEWTAG}"
530	) && touch "${IMPORTDONEFILE}"
531}
532
533cvsmerge()
534{
535
536	cd "${TZDISTDIR}" || exit 1
537	if [ -e "${MERGEDONEFILE}" ]; then
538		cat >&2 <<EOF
539The CVS merge has already been performed.
540EOF
541		return 0
542	fi
543	DOIT cvs -d "${CVSROOT}" update -j"${CVSOLDTAG}" -j"${CVSNEWTAG}" \
544	&& touch "${MERGEDONEFILE}"
545}
546
547resolveconflicts()
548{
549	cd "${TZDISTDIR}" || exit 1
550	if grep -l '^[<=>][<=>][<=>]' *
551	then
552		cat <<EOF
553There appear to be conflicts in the files listed above.
554Resolve conflicts, then re-run this script.
555EOF
556		return 1
557	fi
558}
559
560cvscommitmerge()
561{
562	cd "${TZDISTDIR}" || exit 1
563	if grep -l '^[<=>][<=>][<=>]' *
564	then
565		cat >&2 <<EOF
566There still appear to be conflicts in the files listed above.
567Not performing cvs commit.
568EOF
569		return 1
570	fi
571	if [ -e "${COMMITMERGEDONEFILE}" ]; then
572		cat >&2 <<EOF
573The CVS commmit (of the merge result) has already been performed.
574EOF
575		return 0
576	fi
577	DOIT cvs -d "${CVSROOT}" commit -m "Merge tzdata${NEWVER}" \
578	&& touch "${COMMITMERGEDONEFILE}"
579}
580
581extra()
582{
583	cat <<EOF
584Also do the following:
585 * Edit and commit  src/doc/3RDPARTY
586 * Edit and commit  src/doc/CHANGES
587 * Edit and commit  src/distrib/sets/lists/base/mi
588 *      if the set of installed files altered.
589 * Submit pullup requests for all active release branches.
590 * rm -rf ${WORK_PFX}  (optional)
591 * Verify that
592  ${UPDATE_FROM}
593 * no longer exists.
594EOF
595}
596
597main()
598{
599	set -e
600	validate_pwd
601	findcvsroot
602	mkworkpfx
603	argparse "$@"
604	setup_versions
605	mkworkdir
606	fetch
607	checksig
608	extract
609	addnews
610	trimnews
611	mkimportmsg
612	editimportmsg
613	cvsimport
614	cvsmerge
615	resolveconflicts
616	cvscommitmerge
617	rm -f "${UPDATE_FROM}"
618	extra
619}
620
621main "$@"
622