'''Read a Linux Software Map file

Copyright 1999,2002 by Daniel Fandrich <dan@coneharvesters.com>

$Id: lsm.py,v 1.6 2002/05/14 05:43:42 dan Exp $
'''
# Useful definitions in this file:
#	CanonicalLSM
#	LSM
#	fileurl
#	fixwhitespace
#	stripleadingwhitespace
#
# To do:
#  Convert date to canonical time_t format (at noon) in CanonicalLSM
#  (add a new tag for this)

# $Log: lsm.py,v $
# Revision 1.6  2002/05/14 05:43:42  dan
# Fixed bug in handling filesize in some Begin2 LSM files.
# Strip trailing blank lines in Begin2 LSM files.
#
# Revision 1.5  2000/06/05 20:18:40  dan
# Removed redundant import lists
#
# Revision 1.4  2000/06/05 19:24:16  dan
# Added begin tag to converted LSM dict. Fixed GB and TB overflows.
# Added support for Begin4 style LSM files.
#
# Revision 1.3  1999/03/27 19:51:30  dan
# Change to use new splitfirst semantics. Better handle bad or empty LSM
# files. Simplify Begin2 LSM handler. Use urllib in urljoin.
#
# Revision 1.2  1999/02/26 07:49:22  dan
# Changed safedict to SafeDict
#
# Revision 1.1  1999/02/26 07:04:59  dan
# Initial revision
#

#  This program is free software; you can redistribute it and/or modify
#  it under the terms of the GNU General Public License as published by
#  the Free Software Foundation; either version 2, or (at your option)
#  any later version.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
#
#  You should have received a copy of the GNU General Public License
#  along with this program; if not, write to the Free Software
#  Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.

from string import strip,split,join,lower,upper,atoi, \
                   whitespace,digits,expandtabs,find
import sys
import regex
import urllib
from danutil import span,splitfirst,SafeDict

TAB_WIDTH = 8

BadSite = 'BadSite'	# exception

def fileurl(site):
	'''Given the contents of LSM['primary-site'], returns a list of tuples
	containing the URL and size (if known) for the referenced file(s).
	If the file size is not known, -1 is returned as the file size.
	'''
	urls = []
	baseurl = ''
	savedit = 1
	for l in site:
		pieces = split(strip(l))
		if len(pieces) == 1:
			if not savedit:
				# This isn't the first and only line, so store the last
				# baseurl
				urls.append((baseurl,-1))
				savedit = 1
			if pieces[0][:5] == 'http:' or \
			   pieces[0][:5] == 'file:' or \
			   pieces[0][:4] == 'ftp:':
				# Assume the file is already a URL
				baseurl = pieces[0]
				savedit = 0
			else:
				# e.g. 'sunsite.unc.edu:/pub/Linux/foo'
				host,path = splitfirst(pieces[0],':')
				if path:
					baseurl = urljoin('ftp://' + host, path)
					savedit = 1
				else:
					# Looks like a file name without the optional file size
					# Add it to the base location
					# e.g. 'foofile-0.12.tar.gz'
					if not baseurl:
						raise BadSite	# No base URL given
					url = urljoin(baseurl, pieces[0])
					urls.append((url, -1))
					savedit = 1

			if len(site) == 1:
				# If this is the only entry in the site list, it must be
				# a complete URL, not just the base
				# This is actually not allowed by the LSM spec
				# e.g. 'sunsite.unc.edu:/pub/Linux/foo/foofile-0.12.tar.gz'
				urls.append((baseurl,-1))
				savedit = 1		# This doesn't really matter in this case
		elif len(pieces) == 2:
			if pieces[0][0] in digits:
				# Looks like a size and a file; add it to the base location
				# e.g. '200KB foofile-0.12.tar.gz'
				if not baseurl:
					raise BadSite	# No base URL given
				size = decodesize(pieces[0])
				urls.append((urljoin(baseurl, pieces[1]), size))
				savedit = 1
			else:
				# Maybe this should raise BadSite instead
				baseurl = urljoin('ftp://' + pieces[0], pieces[1])
				savedit = 0
		elif len(pieces) == 3:
			if pieces[0][0] in digits:
				# Looks like a size and a file; add it to the base location
				# e.g. '200 KB foofile-0.12.tar.gz'
				if not baseurl:
					raise BadSite	# No base URL given
				size = decodesize(pieces[0]+pieces[1])
				urls.append((urljoin(baseurl, pieces[2]), size))
				savedit = 1
			else:
				raise BadSite		# Too many arguments
		else:
			raise BadSite		# Too many arguments

	if not savedit:
		# So store the last baseurl
		urls.append((baseurl,-1))

	return urls

def CanonicalLSM(file):
	'''Function to read a Linux Software Map file. Converts old format files
	to look like new (type 3 or 4) format files.'''
	lsm = LSM(file)
	if not lsm.has_key('begin'):
		return {}	# Empty LSM

	# See if LSM is already in canonical format
	if lsm['begin'][0] in ['Begin3', 'Begin4']:
		return lsm

	# Make sure LSM is of a type we can convert to canonical format
	# (we can only convert the Begin2 format right now).
	if lsm['begin'][0] != 'Begin2':
		return {}	# Unknown LSM type

	# Keys with a one-to-one correspondance
	convert_dict = {
	  'title':'title',
	  'version':'version',
	  'required':'platforms',
	  'copypolicy':'copying-policy',
	  'keywords':'keywords',
	  'entered':'entered-date'}
	newlsm = {}

	# Convert all keys with a one-to-one correspondance
	for key in convert_dict.keys():
		if lsm.has_key(key):
			newlsm[convert_dict[key]] = lsm[key]

	# Convert all keys that need some manipulation
	safelsm = SafeDict(lsm,'')

	# description
	newlsm['description'] = safelsm['desc']
	if safelsm.has_key('comment'):
		newlsm['description'] = deletetrailingblanklines(
				newlsm['description'] + [''] + safelsm['comment'])
	if safelsm.has_key('relfiles'):
		newlsm['description'] = deletetrailingblanklines(
				newlsm['description'] + [''] + safelsm['relfiles'])

	# author
	newlsm['author'] = safelsm['authoremail']
	if safelsm.has_key('author'):
		newlsm['author'] = [newlsm['author'][0]+' ('+safelsm['author'][0]+')']

	# maintained-by
	newlsm['maintained-by'] = safelsm['maintemail']
	if safelsm.has_key('author'):
		newlsm['maintained-by'] = [newlsm['maintained-by'][0]+' ('+safelsm['maintainer'][0]+')']

	# primary-site
	newlsm['primary-site'] = [safelsm['site'][0]+' '+safelsm['path'][0],
							  safelsm['filesize'][0]+' '+safelsm['file'][0]]

	# alternate-site
	if len(safelsm['site']) > 1:
		if len(safelsm['filesize']) > 1:
			newlsm['alternate-site'] = \
				[safelsm['site'][1]+' '+safelsm['path'][1],
				 safelsm['filesize'][1]+' '+safelsm['file'][1]]
		else:
			newlsm['alternate-site'] = \
				[safelsm['site'][1]+' '+safelsm['path'][1],
				 safelsm['filesize'][0]+' '+safelsm['file'][0]]

	# Now tag this converted dict as the canonical type
	newlsm['begin'] = ['Begin3']

	return newlsm

def LSM(file):
	'''Function to read a Linux Software Map file
	The function is given a open file handle for the .lsm file.
	It immediately reads the contents into a dictionary tagged by lower-case
	LSM entry name. Each dictionary entry is as a list of strings with the
	trailing \n stripped off, but including the leading whitespace.
	e.g.
	  lsm = LSM(sys.stdin)
	  print 'Title:',lsm['title'][0]
	'''
	line = '\n'		# non-None value that will be ignored
	while line:
		line = file.readline()
		l = strip(line)
		if l == 'Begin3' or l == 'Begin4':
			contents = ReadLsm3(file)
			contents['begin'] = [l]
			return contents
		elif l == "Begin2":
			contents = ReadLsm2(file)
			contents['begin'] = [l]
			return contents

		# Add a pseudo-tag specifying the LSM version
	return {}	# No LSM entry in file

def ReadLsm2(file):
	'''Read the contents of a old-style LSM file
	File must be at beginning of LSM entry after Begin line'''
	contents = {}
	line = file.readline()
	while line and line != 'End\n':
		# Kill trailing newline
		if line[-1:] == '\n':
			line = line[:-1]

		tag,text = splitfirst(line,'=')
		contents[lower(strip(tag))] = [text]

		line = file.readline()

	tag_re = regex.compile('\([a-zA-Z_]+\)\([0-9]+\)$')

	# Numeric sort function for the list of keys
	def sorttags(tag1,tag2,tag_re=tag_re):
		if tag_re.match(tag1) >= 0:
			num1 = atoi(tag_re.group(2))
		else:
			num1 = 0
		if tag_re.match(tag2) >= 0:
			num2 = atoi(tag_re.group(2))
		else:
			num2 = 0
		return (num1 > num2) - (num1 < num2)

	# Get a list of keys sorted only by the numerical portion
	tags = contents.keys()
	tags.sort(sorttags)

	# Now, create new dict combining all numbered tags into keys without
	# the numbers
	newcontents = SafeDict({},[])
	for tag in tags:
		if tag_re.match(tag) >= 0:
			basetag = tag_re.group(1)
		else:
			basetag = tag
		newcontents[basetag] = newcontents[basetag] + contents[tag]

	# Fix whitespace in tag contents
	for tag in newcontents.keys():
		tagcontent = deletetrailingblanklines(newcontents[tag])
		newcontents[tag] = fixwhitespace(tagcontent)

	return newcontents

def ReadLsm3(file):
	'''Read the contents of a new-style LSM file
	File must be at beginning of LSM entry after Begin line'''
	contents = {}
	tag = ''
	value = []
	line = file.readline()
	while line and line != 'End\n':
		# Kill trailing newline
		if line[-1:] == '\n':
			line = line[:-1]
		if not line:
			line = file.readline()
			continue	# ignore blank line
		if line[0] in whitespace:
			# Continuation of previous tag
			if tag:
				value = value + [line]
			#else: print 'Error: no tag'
		else:
			# New tag
			if tag:
				# This ends the previous tag definition -- store it
				contents[lower(tag)] = fixwhitespace(value)
			tag,value = splitfirst(line,':')
			value = [value]
		line = file.readline()

	# At end of LSM entry
	if tag:
		# This ends the last tag definition -- store it
		contents[lower(tag)] = fixwhitespace(value)
	return contents

def fixwhitespace(strlist):
	'''Strip leading whitespace from a list of lines, using some intelligence.
	The first line has all leading whitespace stripped, while indentation
	in remaining lines is preserved.  Tabs are replaced with spaces.'''
	if not strlist:
		return strlist
	return [strip(strlist[0])] + stripleadingwhitespace(strlist[1:])

def stripleadingwhitespace(strlist):
	'''Strip the same amount of leading whitespace from all strings in a list
	and remove tabs.  This preserves indentation but removes the left margin.'''
	# Find the least number of leading spaces in any line
	minspace = sys.maxint
	untabbedlist = []
	for str in strlist:
		str = expandtabs(str,TAB_WIDTH)
		untabbedlist.append(str)
		# Ignore empty lines
		if str:
			space = span(str,whitespace)
			if space < minspace:
				minspace = space
	if minspace == sys.maxint:
		minspace = 0

	# We've found the line with the least number of spaces in it.
	# Now, go through all lines and kill that many leading spaces.
	return map(lambda str, minspace=minspace: str[minspace:], untabbedlist)

def deletetrailingblanklines(strlist):
	'Delete all the blank lines at the end of the list of strings'
	if not strlist:
		return
	copylist = strlist[:]
	while copylist and not copylist[-1]:
		del copylist[-1]
	return copylist

def joinstrings(s):
	'Joins a list of strings into a single string, stripping whitespace'
	return join(map(lambda x: strip(x),s),' ')

def decodesize(str):
	'''Takes a string representing a size in bytes, optionally ending in 'KB',
	'MB', 'GB' or 'TB' and returns the numerical value in bytes.
	'''
	size = atoi(str[:span(str,digits)])
	if upper(str[-2:]) == 'KB' or upper(str[-1]) == 'K':
		size = size * 1024
	elif upper(str[-2:]) == 'MB' or upper(str[-1]) == 'M':
		size = size * 1024 * 1024
	elif upper(str[-2:]) == 'GB' or upper(str[-1]) == 'G':
		size = size * 1024L * 1024 * 1024
	elif upper(str[-2:]) == 'TB' or upper(str[-1]) == 'T':
		size = size * 1024L * 1024 * 1024 * 1024
	return size

def urljoin(part1,part2):
	'Join two pieces of a URL, inserting or removing a slash where necessary'
	if part1[-1] != '/':
		part1 = part1 + '/'
	return urllib.basejoin(part1,part2)

# Demonstrate LSM by reading an LSM file from stdin
if __name__=='__main__':
	#lsm = LSM(sys.stdin)
	lsm = CanonicalLSM(sys.stdin)
	if not lsm:
		print "Error: Doesn't appear to be a LSM file"
		sys.exit(1)

	keys = lsm.keys()
	keys.sort()
	for tag in keys:
		print '%s:' % tag
		for line in lsm[tag]:
			print '',line

	print
	print 'URLs:'
	for file in fileurl(lsm['primary-site']):
		print ' %s (%d KB)' % (file[0], file[1] / 1024)
