| #! /usr/bin/env python |
| ####################################################################### |
| # Newslist $Revision$ |
| # |
| # Syntax: |
| # newslist [ -a ] |
| # |
| # This is a program to create a directory full of HTML pages |
| # which between them contain links to all the newsgroups available |
| # on your server. |
| # |
| # The -a option causes a complete list of all groups to be read from |
| # the server rather than just the ones which have appeared since last |
| # execution. This recreates the local list from scratch. Use this on |
| # the first invocation of the program, and from time to time thereafter. |
| # When new groups are first created they may appear on your server as |
| # empty groups. By default, empty groups are ignored by the -a option. |
| # However, these new groups will not be created again, and so will not |
| # appear in the server's list of 'new groups' at a later date. Hence it |
| # won't appear until you do a '-a' after some articles have appeared. |
| # |
| # I should really keep a list of ignored empty groups and re-check them |
| # for articles on every run, but I haven't got around to it yet. |
| # |
| # This assumes an NNTP news feed. |
| # |
| # Feel free to copy, distribute and modify this code for |
| # non-commercial use. If you make any useful modifications, let me |
| # know! |
| # |
| # (c) Quentin Stafford-Fraser 1994 |
| # [email protected] [email protected] |
| # # |
| ####################################################################### |
| import sys,nntplib, string, marshal, time, os, posix, string |
| |
| ####################################################################### |
| # Check these variables before running! # |
| |
| # Top directory. |
| # Filenames which don't start with / are taken as being relative to this. |
| topdir='/anfs/qsbigdisc/web/html/newspage' |
| |
| # The name of your NNTP host |
| # eg. |
| # newshost = 'nntp-serv.cl.cam.ac.uk' |
| # or use following to get the name from the NNTPSERVER environment |
| # variable: |
| # newshost = posix.environ['NNTPSERVER'] |
| newshost = 'nntp-serv.cl.cam.ac.uk' |
| |
| # The filename for a local cache of the newsgroup list |
| treefile = 'grouptree' |
| |
| # The filename for descriptions of newsgroups |
| # I found a suitable one at ftp.uu.net in /uunet-info/newgroups.gz |
| # You can set this to '' if you don't wish to use one. |
| descfile = 'newsgroups' |
| |
| # The directory in which HTML pages should be created |
| # eg. |
| # pagedir = '/usr/local/lib/html/newspage' |
| # pagedir = 'pages' |
| pagedir = topdir |
| |
| # The html prefix which will refer to this directory |
| # eg. |
| # httppref = '/newspage/', |
| # or leave blank for relative links between pages: (Recommended) |
| # httppref = '' |
| httppref = '' |
| |
| # The name of the 'root' news page in this directory. |
| # A .html suffix will be added. |
| rootpage = 'root' |
| |
| # Set skipempty to 0 if you wish to see links to empty groups as well. |
| # Only affects the -a option. |
| skipempty = 1 |
| |
| # pagelinkicon can contain html to put an icon after links to |
| # further pages. This helps to make important links stand out. |
| # Set to '' if not wanted, or '...' is quite a good one. |
| pagelinkicon='... <img src="http://pelican.cl.cam.ac.uk/icons/page.xbm"> ' |
| |
| # --------------------------------------------------------------------- |
| # Less important personal preferences: |
| |
| # Sublistsize controls the maximum number of items the will appear as |
| # an indented sub-list before the whole thing is moved onto a different |
| # page. The smaller this is, the more pages you will have, but the |
| # shorter each will be. |
| sublistsize = 4 |
| |
| # That should be all. # |
| ####################################################################### |
| |
| for dir in os.curdir, os.environ['HOME']: |
| rcfile = os.path.join(dir, '.newslistrc.py') |
| if os.path.exists(rcfile): |
| print rcfile |
| execfile(rcfile) |
| break |
| |
| from nntplib import NNTP |
| from stat import * |
| |
| rcsrev = '$Revision$' |
| rcsrev = string.join(filter(lambda s: '$' not in s, string.split(rcsrev))) |
| desc = {} |
| |
| # Make (possibly) relative filenames into absolute ones |
| treefile = os.path.join(topdir,treefile) |
| descfile = os.path.join(topdir,descfile) |
| page = os.path.join(topdir,pagedir) |
| |
| # First the bits for creating trees --------------------------- |
| |
| # Addtotree creates/augments a tree from a list of group names |
| def addtotree(tree, groups): |
| print 'Updating tree...' |
| for i in groups: |
| parts = string.splitfields(i,'.') |
| makeleaf(tree, parts) |
| |
| # Makeleaf makes a leaf and the branch leading to it if necessary |
| def makeleaf(tree,path): |
| j = path[0] |
| l = len(path) |
| |
| if not tree.has_key(j): |
| tree[j] = {} |
| if l == 1: |
| tree[j]['.'] = '.' |
| if l > 1: |
| makeleaf(tree[j],path[1:]) |
| |
| # Then the bits for outputting trees as pages ---------------- |
| |
| # Createpage creates an HTML file named <root>.html containing links |
| # to those groups beginning with <root>. |
| |
| def createpage(root, tree, p): |
| filename = os.path.join(pagedir,root+'.html') |
| if root == rootpage: |
| detail = '' |
| else: |
| detail = ' under ' + root |
| f = open(filename,'w') |
| # f.write('Content-Type: text/html\n') |
| f.write('<TITLE>Newsgroups available' + detail + '</TITLE>\n') |
| f.write('<H1>Newsgroups available' + detail +'</H1>\n') |
| f.write('<A HREF="'+httppref+rootpage+'.html">Back to top level</A><P>\n') |
| printtree(f,tree,0,p) |
| f.write('<I>This page automatically created by \'newslist\' v. '+rcsrev+'.') |
| f.write(time.ctime(time.time()) + '</I><P>') |
| f.close() |
| |
| # Printtree prints the groups as a bulleted list. Groups with |
| # more than <sublistsize> subgroups will be put on a separate page. |
| # Other sets of subgroups are just indented. |
| |
| def printtree(f, tree, indent, p): |
| global desc |
| l = len(tree) |
| |
| if l > sublistsize and indent>0: |
| # Create a new page and a link to it |
| f.write('<LI><B><A HREF="'+httppref+p[1:]+'.html">') |
| f.write(p[1:]+'.*') |
| f.write('</A></B>'+pagelinkicon+'\n') |
| createpage(p[1:], tree, p) |
| return |
| |
| kl = tree.keys() |
| |
| if l > 1: |
| kl.sort() |
| if indent > 0: |
| # Create a sub-list |
| f.write('<LI>'+p[1:]+'\n<UL>') |
| else: |
| # Create a main list |
| f.write('<UL>') |
| indent = indent + 1 |
| |
| for i in kl: |
| if i == '.': |
| # Output a newsgroup |
| f.write('<LI><A HREF="news:' + p[1:] + '">'+ p[1:] + '</A> ') |
| if desc.has_key(p[1:]): |
| f.write(' <I>'+desc[p[1:]]+'</I>\n') |
| else: |
| f.write('\n') |
| else: |
| # Output a hierarchy |
| printtree(f,tree[i], indent, p+'.'+i) |
| |
| if l > 1: |
| f.write('\n</UL>') |
| |
| # Reading descriptions file --------------------------------------- |
| |
| # This returns an array mapping group name to its description |
| |
| def readdesc(descfile): |
| global desc |
| |
| desc = {} |
| |
| if descfile == '': |
| return |
| |
| try: |
| d = open(descfile, 'r') |
| print 'Reading descriptions...' |
| except (IOError): |
| print 'Failed to open description file ' + descfile |
| return |
| l = d.readline() |
| while l != '': |
| bits = string.split(l) |
| try: |
| grp = bits[0] |
| dsc = string.join(bits[1:]) |
| if len(dsc)>1: |
| desc[grp] = dsc |
| except (IndexError): |
| pass |
| l = d.readline() |
| |
| # Check that ouput directory exists, ------------------------------ |
| # and offer to create it if not |
| |
| def checkopdir(pagedir): |
| if not os.path.isdir(pagedir): |
| print 'Directory '+pagedir+' does not exist.' |
| print 'Shall I create it for you? (y/n)' |
| if sys.stdin.readline()[0] == 'y': |
| try: |
| os.mkdir(pagedir,0777) |
| except: |
| print 'Sorry - failed!' |
| sys.exit(1) |
| else: |
| print 'OK. Exiting.' |
| sys.exit(1) |
| |
| # Read and write current local tree ---------------------------------- |
| |
| def readlocallist(treefile): |
| print 'Reading current local group list...' |
| tree = {} |
| try: |
| treetime = time.localtime(os.stat(treefile)[ST_MTIME]) |
| except: |
| print '\n*** Failed to open local group cache '+treefile |
| print 'If this is the first time you have run newslist, then' |
| print 'use the -a option to create it.' |
| sys.exit(1) |
| treedate = '%02d%02d%02d' % (treetime[0] % 100 ,treetime[1], treetime[2]) |
| try: |
| dump = open(treefile,'r') |
| tree = marshal.load(dump) |
| dump.close() |
| except (IOError): |
| print 'Cannot open local group list ' + treefile |
| return (tree, treedate) |
| |
| def writelocallist(treefile, tree): |
| try: |
| dump = open(treefile,'w') |
| groups = marshal.dump(tree,dump) |
| dump.close() |
| print 'Saved list to '+treefile+'\n' |
| except: |
| print 'Sorry - failed to write to local group cache '+treefile |
| print 'Does it (or its directory) have the correct permissions?' |
| sys.exit(1) |
| |
| # Return list of all groups on server ----------------------------- |
| |
| def getallgroups(server): |
| print 'Getting list of all groups...' |
| treedate='010101' |
| info = server.list()[1] |
| groups = [] |
| print 'Processing...' |
| if skipempty: |
| print '\nIgnoring following empty groups:' |
| for i in info: |
| grpname = string.split(i[0])[0] |
| if skipempty and string.atoi(i[1]) < string.atoi(i[2]): |
| print grpname+' ', |
| else: |
| groups.append(grpname) |
| print '\n' |
| if skipempty: |
| print '(End of empty groups)' |
| return groups |
| |
| # Return list of new groups on server ----------------------------- |
| |
| def getnewgroups(server, treedate): |
| print 'Getting list of new groups since start of '+treedate+'...', |
| info = server.newgroups(treedate,'000001')[1] |
| print 'got %d.' % len(info) |
| print 'Processing...', |
| groups = [] |
| for i in info: |
| grpname = string.split(i)[0] |
| groups.append(grpname) |
| print 'Done' |
| return groups |
| |
| # Now the main program -------------------------------------------- |
| |
| def main(): |
| global desc |
| |
| tree={} |
| |
| # Check that the output directory exists |
| checkopdir(pagedir); |
| |
| try: |
| print 'Connecting to '+newshost+'...' |
| if sys.version[0] == '0': |
| s = NNTP.init(newshost) |
| else: |
| s = NNTP(newshost) |
| connected = 1 |
| except (nntplib.error_temp, nntplib.error_perm), x: |
| print 'Error connecting to host:', x |
| print 'I\'ll try to use just the local list.' |
| connected = 0 |
| |
| # If -a is specified, read the full list of groups from server |
| if connected and len(sys.argv) > 1 and sys.argv[1] == '-a': |
| |
| groups = getallgroups(s) |
| |
| # Otherwise just read the local file and then add |
| # groups created since local file last modified. |
| else: |
| |
| (tree, treedate) = readlocallist(treefile) |
| if connected: |
| groups = getnewgroups(s, treedate) |
| |
| if connected: |
| addtotree(tree, groups) |
| writelocallist(treefile,tree) |
| |
| # Read group descriptions |
| readdesc(descfile) |
| |
| print 'Creating pages...' |
| createpage(rootpage, tree, '') |
| print 'Done' |
| |
| if __name__ == "__main__": |
| main() |
| |
| # That's all folks |
| ###################################################################### |
