Difference between revisions of "User:WindPower/RCNotify.py"

From Team Fortress Wiki
Jump to: navigation, search
m (Running this script)
m (shonkier business)
 
(13 intermediate revisions by the same user not shown)
Line 1: Line 1:
 
== About this script ==
 
== About this script ==
 
This is a [http://python.org Python] script that:
 
This is a [http://python.org Python] script that:
* Fetches the recent changes log every '''{{User:WindPower/Template:RCNotifyRefreshRate}}''' seconds
+
* Fetches the recent changes log every '''{{User:WindPower/RCNotifyRefreshRate}}''' seconds
* Fetches the [http://www.teamfortress.com TF2 blog] to look for updates
+
* Fetches the [http://www.teamfortress.com/ TF2 blog] to look for updates
 +
* Fetches the [http://store.steampowered.com/news/posts/?feed=steam_updates&appids=440 TF2 news feed] to look for updates
 
* Sends the results back to the IRC bot (#tfwiki on irc.freenode.net), and the bot notifies everyone in the channel about it.
 
* Sends the results back to the IRC bot (#tfwiki on irc.freenode.net), and the bot notifies everyone in the channel about it.
 +
 
== Running this script ==
 
== Running this script ==
* '''WON'T WORK YET, WILL UPDATE SOON'''
 
 
* First way: Download Python and run the script as is.
 
* First way: Download Python and run the script as is.
 
** [http://python.org/download/releases/2.6.6/ Download Python] and install it if you haven't got it yet.
 
** [http://python.org/download/releases/2.6.6/ Download Python] and install it if you haven't got it yet.
** Copy the code in the [[#Source|Source]] section below, and save it in a text file. Give it a .py extension, preferably.
+
** [http://upload.gaiatools.com/files/rcNotify_2.7z Download the archive file] and extract it somewhere. You should have a file named <code>rcNotify.py</code> (the Python script), a file named <code>rcConfig.py</code> (configuration file) and a folder called <code>wikitools</code> (Wiki API library).
** [http://code.google.com/p/python-wikitools/downloads/detail?name=wikitools-1.1.1.zip Download wikitools] and extract the wikitools folder inside the same directory as the script. Rename the folder to "wikitools".
+
** Edit <code>rcConfig.py</code> to your liking. Read the comments next to each line. Do not use Notepad to edit the file, because it cannot read Unix-style linebreaks (<code>\n</code>). Use WordPad or whatever code editor you have.
 
** Open a command prompt or terminal window, and run Python with the script's path as argument.
 
** Open a command prompt or terminal window, and run Python with the script's path as argument.
 
*** On Windows: <code>C:\Python26\python.exe C:\path\to\script.py</code>
 
*** On Windows: <code>C:\Python26\python.exe C:\path\to\script.py</code>
 
*** On Windows: Double-click the file
 
*** On Windows: Double-click the file
 
**** You can also rename it to .pyw and double-click it if you don't want the command prompt window to appear.
 
**** You can also rename it to .pyw and double-click it if you don't want the command prompt window to appear.
*** On Unix (Mac/Linux): <code>python /path/to/script.pt</code>
+
*** On Unix (Mac/Linux): <code>python /path/to/script.py</code>
 
*** On Unix (Mac/Linux), you can also type simply <code>/path/to/script.py</code> if you set the executable bit on the script file beforehand (<code>chmod +x /path/to/script.py</code>).
 
*** On Unix (Mac/Linux), you can also type simply <code>/path/to/script.py</code> if you set the executable bit on the script file beforehand (<code>chmod +x /path/to/script.py</code>).
 
* Second way (Windows only): Download the exe.
 
* Second way (Windows only): Download the exe.
Line 37: Line 38:
 
import time #, Dr. Freeman?
 
import time #, Dr. Freeman?
 
import urllib, urllib2 # Series of tubes
 
import urllib, urllib2 # Series of tubes
 +
import re # Dem regexes
 
import hashlib # Yummy
 
import hashlib # Yummy
 
import pprint # prettyPrint
 
import pprint # prettyPrint
 +
import wikitools # Wiki bindings
 +
import sys # Command-line arguments parsing
 
import traceback # Print error stack traces
 
import traceback # Print error stack traces
import wikitools # Wiki bindings
+
 
 +
# Config:
 +
from rcConfig import *
  
 
# Constants:
 
# Constants:
wiki = wikitools.wiki.Wiki('http://wiki.teamfortress.com/w/api.php')
+
wiki = wikitools.wiki.Wiki(config['wikiUrl'])
refreshRatePage = 'User:WindPower/Template:RCNotifyRefreshRate'
+
refreshRatePage = config['refreshRatePage']
notifyUrl = 'http://irc.biringa.com/tfwikirc.php?shapass=f1ece0751b2d520c2038c121d91db7a2689f74b9'
+
notifyUrl = config['notifyUrl']
 +
httpTimeout = config['httpTimeout']
 
tf2Blog = 'http://www.teamfortress.com/'
 
tf2Blog = 'http://www.teamfortress.com/'
 +
tf2UpdatesFeed = 'http://store.steampowered.com/news/posts/?feed=steam_updates&appids=440'
 +
tf2schemaFeed = 'http://git.optf2.com/schema-tracking/atom/?h=master'
 
# Globals:
 
# Globals:
refreshRate = 180 # 3 minutes by default
+
refreshRate = config['refreshRate'] # 3 minutes by default
 
tf2BlogHash = None # Will be populated later
 
tf2BlogHash = None # Will be populated later
 +
tf2UpdateID = None # Will be populated later
 +
tf2SchemaID = None # Will be populated later
 
lastRC = -1 # Will be populated later
 
lastRC = -1 # Will be populated later
  
Line 74: Line 85:
 
return s
 
return s
 
def urlEncode(s):
 
def urlEncode(s):
return urllib2.quote(urllib2.unquote(eval(u(s).encode('utf8').__repr__().replace('\\x', '%'))))
+
encoded = urllib2.quote(urllib2.unquote(eval(u(s).encode('utf8').__repr__().replace('\\x', '%'))))
 +
encoded = encoded.replace('head%20', 'head_')
 +
encoded = encoded.replace('%7B', '').replace('%7D', '')
 +
return encoded
 
def getNotifyResponse(params):
 
def getNotifyResponse(params):
global notifyUrl
+
global notifyUrl, config, httpTimeout
 
if type(params) in (type(''), type(u'')):
 
if type(params) in (type(''), type(u'')):
 
params = u(params)
 
params = u(params)
 
else:
 
else:
 
params = u(urllib.urlencode(params))
 
params = u(urllib.urlencode(params))
return urllib2.urlopen(notifyUrl, params).read(-1)
+
if config['via'] is not None:
 +
params += '&via=' + u(urlEncode(config['via']))
 +
response = urllib2.urlopen(notifyUrl, params, timeout=httpTimeout).read(-1)
 +
print 'Response:', response
 +
return response
 
def updateRefreshRate():
 
def updateRefreshRate():
global wiki, refreshRate, refreshRatePage
+
global wiki, refreshRate, refreshRatePage, config
 +
if type(config['refreshRate']) is type(0):
 +
return
 
try:
 
try:
 
refreshRate = int(wikitools.page.Page(wiki, refreshRatePage).getWikiText())
 
refreshRate = int(wikitools.page.Page(wiki, refreshRatePage).getWikiText())
 
except:
 
except:
 +
refreshRate = config['refreshRate']
 +
if type(refreshRate) is not type(0):
 
print 'Error while grabbing refresh rate; defaulting to 180s.'
 
print 'Error while grabbing refresh rate; defaulting to 180s.'
 
refreshRate = 180
 
refreshRate = 180
Line 99: Line 121:
 
lastRC = -1
 
lastRC = -1
 
def updateTf2Blog():
 
def updateTf2Blog():
global tf2Blog, tf2BlogHash
+
global tf2Blog, tf2BlogHash, httpTimeout
 
newHash = hashlib.md5()
 
newHash = hashlib.md5()
for i in urllib2.urlopen(tf2Blog):
+
for i in urllib2.urlopen(tf2Blog, timeout=httpTimeout):
 
newHash.update(i)
 
newHash.update(i)
 
newHash = newHash.hexdigest().lower()
 
newHash = newHash.hexdigest().lower()
 
print 'Old hash is', tf2BlogHash, '; New one is', newHash
 
print 'Old hash is', tf2BlogHash, '; New one is', newHash
if tf2BlogHash is None:
+
if tf2BlogHash is None or tf2BlogHash != newHash:
tf2BlogHash = newHash
 
elif tf2BlogHash != newHash:
 
 
tf2BlogHash = newHash
 
tf2BlogHash = newHash
 
print 'TF2 hash response:', getNotifyResponse('newblog=' + tf2BlogHash)
 
print 'TF2 hash response:', getNotifyResponse('newblog=' + tf2BlogHash)
updateLastRC()
+
def checkTf2Update():
updateTf2Blog()
+
global tf2UpdatesFeed, tf2UpdateID, httpTimeout
print 'Started with last RCID =', lastRC
+
idRegex = re.compile(r'<div[^<>]*class="posttitle"[^<>]*><a[^<>]*href="http://store.steampowered.com/news/(\d+)', re.IGNORECASE)
print 'TF2 blog hash =', tf2BlogHash
+
try:
 
+
webfeed = urllib2.urlopen(tf2UpdatesFeed, timeout=httpTimeout).read()
 +
idResult = int(idRegex.search(webfeed).group(1))
 +
except:
 +
print 'Couldn\'t grab latest news ID'
 +
return
 +
if idResult != tf2UpdateID:
 +
tf2UpdateID = idResult
 +
getNotifyResponse('newnews=' + str(idResult))
 +
def checkSchemaUpdate():
 +
global tf2schemaFeed, tf2SchemaID, httpTimeout
 +
idRegex = re.compile(r'<id>\s*(.*?)\s*</id>', re.IGNORECASE)
 +
try:
 +
webfeed = urllib2.urlopen(tf2schemaFeed, timeout=httpTimeout).read()
 +
idResult = idRegex.search(webfeed).group(1)
 +
except:
 +
print 'Couldn\'t grab latest schema'
 +
return
 +
if idResult != tf2SchemaID:
 +
tf2SchemaID = idResult
 +
getNotifyResponse('newschema=' + str(idResult))
 
def reviewRC(rc):
 
def reviewRC(rc):
 
global lastRC
 
global lastRC
Line 144: Line 183:
 
'oldsize': rc['oldlen'],
 
'oldsize': rc['oldlen'],
 
'flags': flag,
 
'flags': flag,
'comment': rc['comment']
+
'comment': rc['comment'],
 +
'timestamp': rc['timestamp']
 
}
 
}
 +
optionalstuff = ('logtype', 'logid', 'logaction')
 +
for o in optionalstuff:
 +
if o in rc:
 +
params[o] = rc[o]
 
return params
 
return params
 
def multiUrlEncode(allParams):
 
def multiUrlEncode(allParams):
Line 156: Line 200:
 
return u'&'.join(s)
 
return u'&'.join(s)
 
def checkForRCs():
 
def checkForRCs():
 +
global config
 
rcs = wikitools.api.APIRequest(wiki, {
 
rcs = wikitools.api.APIRequest(wiki, {
 
'action': 'query',
 
'action': 'query',
 
'list': 'recentchanges',
 
'list': 'recentchanges',
'rclimit': '100',
+
'rclimit': str(config['rcLimit']),
'rcprop': 'user|comment|title|ids|sizes|redirect|flags'
+
'rcprop': 'user|comment|title|ids|timestamp|sizes|redirect|flags|loginfo'
}).query(querycontinue=False)['query']['recentchanges']
+
}).query(querycontinue=False, timeout=15)['query']['recentchanges']
 
rcs.reverse() # Chronological order
 
rcs.reverse() # Chronological order
 
allParams = []
 
allParams = []
Line 168: Line 213:
 
if rc is not None:
 
if rc is not None:
 
allParams.append(rc)
 
allParams.append(rc)
 +
allParams = allParams[:min(config['rcSubmitLimit'], len(allParams))]
 
response = getNotifyResponse(multiUrlEncode(allParams))
 
response = getNotifyResponse(multiUrlEncode(allParams))
 
updateLastRC(response)
 
updateLastRC(response)
def main():
+
def main(once=False):
global refreshRate
+
global refreshRate, config
 +
updateRefreshRate()
 +
updateLastRC()
 +
updateTf2Blog()
 +
print 'Started with last RCID =', lastRC
 +
print 'TF2 blog hash =', tf2BlogHash
 +
print 'Refresh rate =', refreshRate
 +
tf2BlogCountdown = 1 # Start at 1 to ensure the blog gets refreshed on the first run
 +
once = once or '--once' in sys.argv[1:]
 
while True:
 
while True:
 
try:
 
try:
Line 181: Line 235:
 
traceback.print_exc()
 
traceback.print_exc()
 
try:
 
try:
print 'Checking for updates to TF2\'s blog.'
+
tf2BlogCountdown -= 1
updateTf2Blog()
+
if tf2BlogCountdown <= 0:
 +
tf2BlogCountdown = config['blogRefresh']
 +
print 'Checking for updates to TF2\'s blog.'
 +
updateTf2Blog()
 +
print 'Checking for news updates'
 +
checkTf2Update()
 +
print 'Checking for schema updates'
 +
checkSchemaUpdate()
 
except KeyboardInterrupt:
 
except KeyboardInterrupt:
 
raise KeyboardInterrupt
 
raise KeyboardInterrupt
Line 188: Line 249:
 
print 'Error while checking for updates to TF2\'s blog.'
 
print 'Error while checking for updates to TF2\'s blog.'
 
traceback.print_exc()
 
traceback.print_exc()
 +
if once:
 +
print 'Exitting after only one run.'
 +
break
 
print 'Sleeping for', refreshRate, 'seconds.'
 
print 'Sleeping for', refreshRate, 'seconds.'
 
time.sleep(refreshRate)
 
time.sleep(refreshRate)
Line 193: Line 257:
 
print 'End.'
 
print 'End.'
 
break
 
break
main()</nowiki></pre>
+
if __name__ == '__main__':
 +
main()</nowiki></pre>

Latest revision as of 11:24, 12 November 2021

About this script

This is a Python script that:

  • Fetches the recent changes log every 30 seconds
  • Fetches the TF2 blog to look for updates
  • Fetches the TF2 news feed to look for updates
  • Sends the results back to the IRC bot (#tfwiki on irc.freenode.net), and the bot notifies everyone in the channel about it.

Running this script

  • First way: Download Python and run the script as is.
    • Download Python and install it if you haven't got it yet.
    • Download the archive file and extract it somewhere. You should have a file named rcNotify.py (the Python script), a file named rcConfig.py (configuration file) and a folder called wikitools (Wiki API library).
    • Edit rcConfig.py to your liking. Read the comments next to each line. Do not use Notepad to edit the file, because it cannot read Unix-style linebreaks (\n). Use WordPad or whatever code editor you have.
    • Open a command prompt or terminal window, and run Python with the script's path as argument.
      • On Windows: C:\Python26\python.exe C:\path\to\script.py
      • On Windows: Double-click the file
        • You can also rename it to .pyw and double-click it if you don't want the command prompt window to appear.
      • On Unix (Mac/Linux): python /path/to/script.py
      • On Unix (Mac/Linux), you can also type simply /path/to/script.py if you set the executable bit on the script file beforehand (chmod +x /path/to/script.py).
  • Second way (Windows only): Download the exe.
    • Coming soon if there is demand.

Source

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import time #, Dr. Freeman?
import urllib, urllib2 # Series of tubes
import re # Dem regexes
import hashlib # Yummy
import pprint # prettyPrint
import wikitools # Wiki bindings
import sys # Command-line arguments parsing
import traceback # Print error stack traces

# Config:
from rcConfig import *

# Constants:
wiki = wikitools.wiki.Wiki(config['wikiUrl'])
refreshRatePage = config['refreshRatePage']
notifyUrl = config['notifyUrl']
httpTimeout = config['httpTimeout']
tf2Blog = 'http://www.teamfortress.com/'
tf2UpdatesFeed = 'http://store.steampowered.com/news/posts/?feed=steam_updates&appids=440'
tf2schemaFeed = 'http://git.optf2.com/schema-tracking/atom/?h=master'
# Globals:
refreshRate = config['refreshRate'] # 3 minutes by default
tf2BlogHash = None # Will be populated later
tf2UpdateID = None # Will be populated later
tf2SchemaID = None # Will be populated later
lastRC = -1 # Will be populated later

def u(s):
	if type(s) is type(u''):
		return s
	if type(s) is type(''):
		try:
			return unicode(s)
		except:
			try:
				return unicode(s.decode('utf8'))
			except:
				try:
					return unicode(s.decode('windows-1252'))
				except:
					return unicode(s, errors='ignore')
	try:
		return unicode(s)
	except:
		try:
			return u(str(s))
		except:
			return s
def urlEncode(s):
	encoded = urllib2.quote(urllib2.unquote(eval(u(s).encode('utf8').__repr__().replace('\\x', '%'))))
	encoded = encoded.replace('head%20', 'head_')
	encoded = encoded.replace('%7B', '').replace('%7D', '')
	return encoded
def getNotifyResponse(params):
	global notifyUrl, config, httpTimeout
	if type(params) in (type(''), type(u'')):
		params = u(params)
	else:
		params = u(urllib.urlencode(params))
	if config['via'] is not None:
		params += '&via=' + u(urlEncode(config['via']))
	response = urllib2.urlopen(notifyUrl, params, timeout=httpTimeout).read(-1)
	print 'Response:', response
	return response
def updateRefreshRate():
	global wiki, refreshRate, refreshRatePage, config
	if type(config['refreshRate']) is type(0):
		return
	try:
		refreshRate = int(wikitools.page.Page(wiki, refreshRatePage).getWikiText())
	except:
		refreshRate = config['refreshRate']
	if type(refreshRate) is not type(0):
		print 'Error while grabbing refresh rate; defaulting to 180s.'
		refreshRate = 180
def updateLastRC(last=None):
	global lastRC
	try:
		if last is None:
			lastRC = int(getNotifyResponse('requestrcid=1'))
		else:
			lastRC = int(last)
	except:
		lastRC = -1
def updateTf2Blog():
	global tf2Blog, tf2BlogHash, httpTimeout
	newHash = hashlib.md5()
	for i in urllib2.urlopen(tf2Blog, timeout=httpTimeout):
		newHash.update(i)
	newHash = newHash.hexdigest().lower()
	print 'Old hash is', tf2BlogHash, '; New one is', newHash
	if tf2BlogHash is None or tf2BlogHash != newHash:
		tf2BlogHash = newHash
		print 'TF2 hash response:', getNotifyResponse('newblog=' + tf2BlogHash)
def checkTf2Update():
	global tf2UpdatesFeed, tf2UpdateID, httpTimeout
	idRegex = re.compile(r'<div[^<>]*class="posttitle"[^<>]*><a[^<>]*href="http://store.steampowered.com/news/(\d+)', re.IGNORECASE)
	try:
		webfeed = urllib2.urlopen(tf2UpdatesFeed, timeout=httpTimeout).read()
		idResult = int(idRegex.search(webfeed).group(1))
	except:
		print 'Couldn\'t grab latest news ID'
		return
	if idResult != tf2UpdateID:
			tf2UpdateID = idResult
			getNotifyResponse('newnews=' + str(idResult))
def checkSchemaUpdate():
	global tf2schemaFeed, tf2SchemaID, httpTimeout
	idRegex = re.compile(r'<id>\s*(.*?)\s*</id>', re.IGNORECASE)
	try:
		webfeed = urllib2.urlopen(tf2schemaFeed, timeout=httpTimeout).read()
		idResult = idRegex.search(webfeed).group(1)
	except:
		print 'Couldn\'t grab latest schema'
		return
	if idResult != tf2SchemaID:
			tf2SchemaID = idResult
			getNotifyResponse('newschema=' + str(idResult))
def reviewRC(rc):
	global lastRC
	if rc['rcid'] <= lastRC:
		return None
	pprint.PrettyPrinter(indent=4).pprint(rc)
	flag = ''
	if 'redirect' in rc:
		flag += 'R'
	if rc['type'] == u'new':
		flag += 'N'
	elif rc['type'] == u'log':
		flag += 'L'
	if 'minor' in rc:
		flag += 'm'
	if 'bot' in rc:
		flag += 'b'
	if not flag:
		flag = '-'
	params = {
		'rcid': rc['rcid'],
		'user': rc['user'],
		'title': rc['title'],
		'pageid': rc['pageid'],
		'namespace': rc['ns'],
		'newrevid': rc['revid'],
		'oldrevid': rc['old_revid'],
		'newsize': rc['newlen'],
		'oldsize': rc['oldlen'],
		'flags': flag,
		'comment': rc['comment'],
		'timestamp': rc['timestamp']
	}
	optionalstuff = ('logtype', 'logid', 'logaction')
	for o in optionalstuff:
		if o in rc:
			params[o] = rc[o]
	return params
def multiUrlEncode(allParams):
	s = []
	c = 0
	for p in allParams:
		for k in p.keys():
			s.append(urlEncode(k) + u'_' + u(c) + u'=' + urlEncode(p[k]))
		c += 1
	return u'&'.join(s)
def checkForRCs():
	global config
	rcs = wikitools.api.APIRequest(wiki, {
			'action': 'query',
			'list': 'recentchanges',
			'rclimit': str(config['rcLimit']),
			'rcprop': 'user|comment|title|ids|timestamp|sizes|redirect|flags|loginfo'
		}).query(querycontinue=False, timeout=15)['query']['recentchanges']
	rcs.reverse() # Chronological order
	allParams = []
	for rc in rcs:
		rc = reviewRC(rc)
		if rc is not None:
			allParams.append(rc)
	allParams = allParams[:min(config['rcSubmitLimit'], len(allParams))]
	response = getNotifyResponse(multiUrlEncode(allParams))
	updateLastRC(response)
def main(once=False):
	global refreshRate, config
	updateRefreshRate()
	updateLastRC()
	updateTf2Blog()
	print 'Started with last RCID =', lastRC
	print 'TF2 blog hash =', tf2BlogHash
	print 'Refresh rate =', refreshRate
	tf2BlogCountdown = 1 # Start at 1 to ensure the blog gets refreshed on the first run
	once = once or '--once' in sys.argv[1:]
	while True:
		try:
			try:
				print 'Checking for RCs.'
				checkForRCs()
			except:
				print 'Error while checking for RCs.'
				traceback.print_exc()
			try:
				tf2BlogCountdown -= 1
				if tf2BlogCountdown <= 0:
					tf2BlogCountdown = config['blogRefresh']
					print 'Checking for updates to TF2\'s blog.'
					updateTf2Blog()
					print 'Checking for news updates'
					checkTf2Update()
					print 'Checking for schema updates'
					checkSchemaUpdate()
			except KeyboardInterrupt:
				raise KeyboardInterrupt
			except:
				print 'Error while checking for updates to TF2\'s blog.'
				traceback.print_exc()
			if once:
				print 'Exitting after only one run.'
				break
			print 'Sleeping for', refreshRate, 'seconds.'
			time.sleep(refreshRate)
		except KeyboardInterrupt:
			print 'End.'
			break
if __name__ == '__main__':
	main()