| #! /usr/bin/env python |
| ####################################################################### |
| # Newslist $Revision$ |
| # |
| # Syntax: |
| # newslist [ -a ] |
| # |
| # This is a program to create a directory full of HTML pages |
| # which between them contain links to all the newsgroups available |
| # on your server. |
| # |
| # The -a option causes a complete list of all groups to be read from |
| # the server rather than just the ones which have appeared since last |
| # execution. This recreates the local list from scratch. Use this on |
| # the first invocation of the program, and from time to time thereafter. |
| # When new groups are first created they may appear on your server as |
| # empty groups. By default, empty groups are ignored by the -a option. |
| # However, these new groups will not be created again, and so will not |
| # appear in the server's list of 'new groups' at a later date. Hence it |
| # won't appear until you do a '-a' after some articles have appeared. |
| # |
| # I should really keep a list of ignored empty groups and re-check them |
| # for articles on every run, but I haven't got around to it yet. |
| # |
| # This assumes an NNTP news feed. |
| # |
| # Feel free to copy, distribute and modify this code for |
| # non-commercial use. If you make any useful modifications, let me |
| # know! |
| # |
| # (c) Quentin Stafford-Fraser 1994 |
| # [email protected] [email protected] |
| # # |
| ####################################################################### |
| import sys, nntplib, marshal, time, os |
| |
| ####################################################################### |
| # Check these variables before running! # |
| |
| # Top directory. |
| # Filenames which don't start with / are taken as being relative to this. |
| topdir = os.path.expanduser('~/newspage') |
| |
| # The name of your NNTP host |
| # eg. |
| # newshost = 'nntp-serv.cl.cam.ac.uk' |
| # or use following to get the name from the NNTPSERVER environment |
| # variable: |
| # newshost = os.environ['NNTPSERVER'] |
| newshost = 'news.example.com' |
| |
| # The filename for a local cache of the newsgroup list |
| treefile = 'grouptree' |
| |
| # The filename for descriptions of newsgroups |
| # I found a suitable one at ftp.uu.net in /uunet-info/newgroups.gz |
| # You can set this to '' if you don't wish to use one. |
| descfile = 'newsgroups' |
| |
| # The directory in which HTML pages should be created |
| # eg. |
| # pagedir = '/usr/local/lib/html/newspage' |
| # pagedir = 'pages' |
| pagedir = topdir |
| |
| # The html prefix which will refer to this directory |
| # eg. |
| # httppref = '/newspage/', |
| # or leave blank for relative links between pages: (Recommended) |
| # httppref = '' |
| httppref = '' |
| |
| # The name of the 'root' news page in this directory. |
| # A .html suffix will be added. |
| rootpage = 'root' |
| |
| # Set skipempty to 0 if you wish to see links to empty groups as well. |
| # Only affects the -a option. |
| skipempty = 1 |
| |
| # pagelinkicon can contain html to put an icon after links to |
| # further pages. This helps to make important links stand out. |
| # Set to '' if not wanted, or '...' is quite a good one. |
| pagelinkicon = '... <img src="http://pelican.cl.cam.ac.uk/icons/page.xbm"> ' |
| |
| # --------------------------------------------------------------------- |
| # Less important personal preferences: |
| |
| # Sublistsize controls the maximum number of items the will appear as |
| # an indented sub-list before the whole thing is moved onto a different |
| # page. The smaller this is, the more pages you will have, but the |
| # shorter each will be. |
| sublistsize = 4 |
| |
| # That should be all. # |
| ####################################################################### |
| |
| for dir in os.curdir, os.environ['HOME']: |
| rcfile = os.path.join(dir, '.newslistrc.py') |
| if os.path.exists(rcfile): |
| print rcfile |
| execfile(rcfile) |
| break |
| |
| from nntplib import NNTP |
| from stat import * |
| |
| rcsrev = '$Revision$' |
| rcsrev = ' '.join(filter(lambda s: '$' not in s, rcsrev.split())) |
| desc = {} |
| |
| # Make (possibly) relative filenames into absolute ones |
| treefile = os.path.join(topdir,treefile) |
| descfile = os.path.join(topdir,descfile) |
| page = os.path.join(topdir,pagedir) |
| |
| # First the bits for creating trees --------------------------- |
| |
| # Addtotree creates/augments a tree from a list of group names |
| def addtotree(tree, groups): |
| print 'Updating tree...' |
| for i in groups: |
| parts = i.split('.') |
| makeleaf(tree, parts) |
| |
| # Makeleaf makes a leaf and the branch leading to it if necessary |
| def makeleaf(tree,path): |
| j = path[0] |
| l = len(path) |
| |
| if j not in tree: |
| tree[j] = {} |
| if l == 1: |
| tree[j]['.'] = '.' |
| if l > 1: |
| makeleaf(tree[j],path[1:]) |
| |
| # Then the bits for outputting trees as pages ---------------- |
| |
| # Createpage creates an HTML file named <root>.html containing links |
| # to those groups beginning with <root>. |
| |
| def createpage(root, tree, p): |
| filename = os.path.join(pagedir, root+'.html') |
| if root == rootpage: |
| detail = '' |
| else: |
| detail = ' under ' + root |
| with open(filename, 'w') as f: |
| # f.write('Content-Type: text/html\n') |
| f.write('<html>\n<head>\n') |
| f.write('<title>Newsgroups available%s</title>\n' % detail) |
| f.write('</head>\n<body>\n') |
| f.write('<h1>Newsgroups available%s</h1>\n' % detail) |
| f.write('<a href="%s%s.html">Back to top level</a><p>\n' % |
| (httppref, rootpage)) |
| printtree(f, tree, 0, p) |
| f.write('\n<p>') |
| f.write("<i>This page automatically created by 'newslist' v. %s." % |
| rcsrev) |
| f.write(time.ctime(time.time()) + '</i>\n') |
| f.write('</body>\n</html>\n') |
| |
| # Printtree prints the groups as a bulleted list. Groups with |
| # more than <sublistsize> subgroups will be put on a separate page. |
| # Other sets of subgroups are just indented. |
| |
| def printtree(f, tree, indent, p): |
| l = len(tree) |
| |
| if l > sublistsize and indent > 0: |
| # Create a new page and a link to it |
| f.write('<li><b><a href="%s%s.html">' % (httppref, p[1:])) |
| f.write(p[1:] + '.*') |
| f.write('</a></b>%s\n' % pagelinkicon) |
| createpage(p[1:], tree, p) |
| return |
| |
| kl = tree.keys() |
| |
| if l > 1: |
| kl.sort() |
| if indent > 0: |
| # Create a sub-list |
| f.write('<li>%s\n<ul>' % p[1:]) |
| else: |
| # Create a main list |
| f.write('<ul>') |
| indent = indent + 1 |
| |
| for i in kl: |
| if i == '.': |
| # Output a newsgroup |
| f.write('<li><a href="news:%s">%s</a> ' % (p[1:], p[1:])) |
| if p[1:] in desc: |
| f.write(' <i>%s</i>\n' % desc[p[1:]]) |
| else: |
| f.write('\n') |
| else: |
| # Output a hierarchy |
| printtree(f, tree[i], indent, p+'.'+i) |
| |
| if l > 1: |
| f.write('\n</ul>') |
| |
| # Reading descriptions file --------------------------------------- |
| |
| # This returns a dict mapping group name to its description |
| |
| def readdesc(descfile): |
| global desc |
| desc = {} |
| |
| if descfile == '': |
| return |
| |
| try: |
| with open(descfile, 'r') as d: |
| print 'Reading descriptions...' |
| for l in d: |
| bits = l.split() |
| try: |
| grp = bits[0] |
| dsc = ' '.join(bits[1:]) |
| if len(dsc) > 1: |
| desc[grp] = dsc |
| except IndexError: |
| pass |
| except IOError: |
| print 'Failed to open description file ' + descfile |
| return |
| |
| # Check that ouput directory exists, ------------------------------ |
| # and offer to create it if not |
| |
| def checkopdir(pagedir): |
| if not os.path.isdir(pagedir): |
| print 'Directory %s does not exist.' % pagedir |
| print 'Shall I create it for you? (y/n)' |
| if sys.stdin.readline()[0] == 'y': |
| try: |
| os.mkdir(pagedir, 0777) |
| except: |
| print 'Sorry - failed!' |
| sys.exit(1) |
| else: |
| print 'OK. Exiting.' |
| sys.exit(1) |
| |
| # Read and write current local tree ---------------------------------- |
| |
| def readlocallist(treefile): |
| print 'Reading current local group list...' |
| tree = {} |
| try: |
| treetime = time.localtime(os.stat(treefile)[ST_MTIME]) |
| except: |
| print '\n*** Failed to open local group cache '+treefile |
| print 'If this is the first time you have run newslist, then' |
| print 'use the -a option to create it.' |
| sys.exit(1) |
| treedate = '%02d%02d%02d' % (treetime[0] % 100, treetime[1], treetime[2]) |
| try: |
| with open(treefile, 'rb') as dump: |
| tree = marshal.load(dump) |
| except IOError: |
| print 'Cannot open local group list ' + treefile |
| return (tree, treedate) |
| |
| def writelocallist(treefile, tree): |
| try: |
| with open(treefile, 'wb') as dump: |
| groups = marshal.dump(tree, dump) |
| print 'Saved list to %s\n' % treefile |
| except: |
| print 'Sorry - failed to write to local group cache', treefile |
| print 'Does it (or its directory) have the correct permissions?' |
| sys.exit(1) |
| |
| # Return list of all groups on server ----------------------------- |
| |
| def getallgroups(server): |
| print 'Getting list of all groups...' |
| treedate = '010101' |
| info = server.list()[1] |
| groups = [] |
| print 'Processing...' |
| if skipempty: |
| print '\nIgnoring following empty groups:' |
| for i in info: |
| grpname = i[0].split()[0] |
| if skipempty and int(i[1]) < int(i[2]): |
| print grpname + ' ', |
| else: |
| groups.append(grpname) |
| print '\n' |
| if skipempty: |
| print '(End of empty groups)' |
| return groups |
| |
| # Return list of new groups on server ----------------------------- |
| |
| def getnewgroups(server, treedate): |
| print 'Getting list of new groups since start of %s...' % treedate, |
| info = server.newgroups(treedate, '000001')[1] |
| print 'got %d.' % len(info) |
| print 'Processing...', |
| groups = [] |
| for i in info: |
| grpname = i.split()[0] |
| groups.append(grpname) |
| print 'Done' |
| return groups |
| |
| # Now the main program -------------------------------------------- |
| |
| def main(): |
| tree = {} |
| |
| # Check that the output directory exists |
| checkopdir(pagedir) |
| |
| try: |
| print 'Connecting to %s...' % newshost |
| if sys.version[0] == '0': |
| s = NNTP.init(newshost) |
| else: |
| s = NNTP(newshost) |
| connected = True |
| except (nntplib.error_temp, nntplib.error_perm), x: |
| print 'Error connecting to host:', x |
| print 'I\'ll try to use just the local list.' |
| connected = False |
| |
| # If -a is specified, read the full list of groups from server |
| if connected and len(sys.argv) > 1 and sys.argv[1] == '-a': |
| groups = getallgroups(s) |
| |
| # Otherwise just read the local file and then add |
| # groups created since local file last modified. |
| else: |
| |
| (tree, treedate) = readlocallist(treefile) |
| if connected: |
| groups = getnewgroups(s, treedate) |
| |
| if connected: |
| addtotree(tree, groups) |
| writelocallist(treefile,tree) |
| |
| # Read group descriptions |
| readdesc(descfile) |
| |
| print 'Creating pages...' |
| createpage(rootpage, tree, '') |
| print 'Done' |
| |
| if __name__ == "__main__": |
| main() |
| |
| # That's all folks |
| ###################################################################### |
