Vision.WebNodes

1 import types, Numeric, re, urllib, string 2 3 from NetworkEditor.items import NetworkNode 4 from mglutil.web.HTMLParser import ParseHTML 5 6

7 -def buildCGIWidget(node, form):

8 for field in form.fieldOrder: 9 inputString = "" 10 argsString = "" 11 name = field[0] 12 dict = eval('form.'+field[1]) 13 14 if type(dict[name]) == types.DictType: 15 dict = dict[name] 16 17 if dict.has_key('value'): 18 value = dict['value'] 19 else: 20 value = None 21 fieldType = field[1] 22 23 if fieldType == 'input': 24 if dict['type'] in ['submit', 'reset', 'password']: 25 print 'INPUT OF TYPE '+dict['type']+\ 26 ' NOT SUPPORTED YET!' 27 continue 28 29 elif dict['type'] in ['text', None]: 30 node.widgetDescr[name] = { 31 'class':'NEEntry', 32 'initialValue':value, 33 'labelCfg':{'text':name,}, 'labelSide':'left', } 34 node.inputPortsDescr.append( {'name':name} ) 35 inputString = inputString + name + ', ' 36 argsString = argsString + name + '=' + name + ', ' 37 38 elif dict['type'] == 'file': 39 node.widgetDescr[name] = { 40 'class':'NEEntryWithFileBrowser', 'title':name, 41 'labelCfg':{'text':name, 'foreground':'blue',}, 42 'labelSide':'left', } 43 node.inputPortsDescr.append( {'name':name} ) 44 inputString = inputString + name + ', ' 45 argsString = argsString + name + '=' + name + ', ' 46 47 elif fieldType == 'hiddenInput': 48 argsString = argsString+name+'=\"'+ value +'\", ' 49 50 51 elif fieldType == 'radiobutton': 52 radioValues = [] 53 for val in dict[name]: 54 radioValues.append(val[0]) 55 node.widgetDescr[name] = { 56 'class':'NERadioButton', 57 'valueList':radioValues} 58 59 node.inputPortsDescr.append( {'name':name} ) 60 inputString = inputString + name + ', ' 61 argsString = argsString + name + '=' + name +', ' 62 63 elif fieldType == 'checkbutton': 64 for checkbox in dict: 65 cstatus = checkbox[1] 66 cvalue = checkbox[0] 67 node.widgetDescr[cvalue] = { 68 'class':'NECheckButton', 69 'initialValue':cvalue, 70 'labelCfg':{'text':cvalue}, 71 'labelSide':'left', 72 'status':cstatus 73 } 74 node.inputPortsDescr.append( {'name':name} ) 75 inputString = inputString + name + ', ' 76 argsString = argsString + name + '=' + name +', ' 77 78 elif fieldType == 'select': 79 options = dict['options'] 80 node.widgetDescr[name] = { 81 'class': 'NEComboBox', 82 'choices':options, 83 'labelCfg':{'text':name}, 'labelSide':'w'} 84 node.inputPortsDescr.append( {'name':name} ) 85 inputString = inputString + name + ', ' 86 argsString = argsString + name + '=' + name + ', ' 87 88 elif fieldType == 'textarea': 89 node.widgetDescr[name] = { 90 'class': 'NEScrolledText', 'hull_width':200, 91 'hull_height':100, 'title':name, 'button':None} 92 node.inputPortsDescr.append( {'name':name} ) 93 inputString = inputString + name + ', ' 94 argsString = argsString + name + '=' + name + ', '

95 96 97

98 -class WebBrowser(NetworkNode):

100 - def __init__(self, name='Web Browser', **kw):

101 kw['name'] = name 102 apply( NetworkNode.__init__, (self,), kw ) 103 104 self.inputPortsDescr.append( { 105 'name':'URL', 'datatype':'string', 'required':False, 106 } ) 107 self.inputPortsDescr.append( { 108 'name':'filename', 'datatype':'string', 'required':False, 109 } ) 110 111 from grail.grailApp import getGrailApp 112 self.browser = getGrailApp() 113 114 self.clientObj = GoogleClient() 115 116 code = """def doit(self, URL, filename): 117 118 if self.inputPorts[0].hasNewData(): 119 self.browser.context.load(URL[0]) 120 elif self.inputPorts[1].hasNewData(): 121 import os 122 self.browser.context.load('file:/'+filename[0]) 123 \n""" 124 125 self.setFunction(code)

126 127 128

129 -class PublicHttpClient:

130

131 - def __init__(self, url):

132 self.url = url 133 self.arguments = {}

134 135

136 - def argCheck(self):

137 return 1

138

139 - def run(self):

140 if self.argCheck() == 0: 141 return 142 args = urllib.urlencode( self.arguments ) 143 f = urllib.urlopen(self.url, args) 144 return f.read()

145 146

147 - def setArguments(self, **kw):

148 for k,v in kw.items(): 149 if k not in self.arguments.keys(): 150 raise RuntimeError (k+' is not a valid argument') 151 152 if type(v) in (types.FloatType, types.IntType, 153 types.LongType, types.StringType): 154 self.arguments[k] = v 155 else: 156 pat = re.compile("[\[,\]\012]") 157 arrayStr = re.sub(pat, '', str(Numeric.array(v).flat) ) 158 c = string.split(arrayStr) 159 c = map( float, c ) 160 c = re.sub(pat, '', str(c) ) 161 self.arguments[k] = c

162 163

164 -class ConsurfClient(PublicHttpClient):

165

166 - def __init__(self):

167 self.urlBase = 'http://consurf.tau.ac.il/' 168 self.cgi = 'cgi-bin/consurf/consurf.cgi' 169 url = self.urlBase + self.cgi 170 PublicHttpClient.__init__(self, url) 171 172 self.arguments = { 173 "pdb_ID": '1crn', 174 "chain": 'none', 175 "ALGORITHM": 'Likelihood', # or "Parsimony" 176 "ITERATIONS": '1', 177 "ESCORE": '0.001', 178 "MAX_NUM_HOMOL": '50' 179 }

180

181 - def getResultPage(self, result):

182 # parses html returned by job submission and returns either the 183 # path to the result page (i.e. results/1032307369/index.htm) or None 184 lines = string.split(result) 185 page = None 186 for l in lines: 187 if l[:18]=="/~consurf/results/": 188 page = l[1:string.find(l,"index.htm")] 189 break 190 return page

191 192

193 - def checkForCompletion(self, page):

194 # opens the result page and check every 30 seconds if the 195 # computation is done by checking is the page contains 196 # "ConSurf finished calculation" 197 url = self.urlBase + page + 'output.html' 198 print 'checkForCompletion', url 199 done = 0 200 count = 0 201 import time 202 while 1: 203 f = urllib.urlopen(url) 204 result = f.read() 205 f.close() 206 count = count + 1 207 print 'checkForCompletion COUNT: ', count 208 done = string.find(result, "ConSurf finished calculation") 209 if done != -1: 210 break 211 if string.find(result, "ERROR") != -1: 212 print "ERROR ********************************************" 213 print result 214 break 215 if count == 40: 216 print "giving up after %d minutes"%(count*0.5,) 217 break 218 time.sleep(30) 219 220 url = self.urlBase + page + 'pdb%s.gradesPE'%self.arguments['pdb_ID'] 221 return url

222 223

224 - def getConservationScores(self, url):

225 f = urllib.urlopen(url) 226 result = f.read() 227 f.close() 228 lines = string.split(result, '\n') 229 scores = [] 230 i = 0 231 for l in lines: 232 if l[-7:]=='VARIETY': 233 break 234 i = i + 1 235 for l in lines[i+1:]: 236 words = string.split(l) 237 if len(words) < 5: continue 238 if words[2] == '-': 239 continue 240 col = string.find(words[2], ':') 241 scores.append( ( words[2][:col], words[2][col+1], 242 float(words[3]), int(words[4]) ) ) 243 return scores

244 245 246 from NetworkEditor.items import NetworkNode 247

248 -class ConsurfClientNode(NetworkNode):

249

250 - def readCache(self, cacheFile):

251 try: 252 f = open(cacheFile) 253 data = f.readlines() 254 f.close() 255 data = map(string.split, data) 256 cache = {} 257 for k,v in data: 258 cache[k] = v 259 except IOError: 260 cache = {} 261 262 return cache

263 264

265 - def saveCache(self, cacheFile):

266 try: 267 f = open(cacheFile, 'w') 268 map( lambda x, f=f: f.write("%s %s\n"%x), self.cache.items()) 269 f.close() 270 except IOError: 271 print 'Could not open cache file %s for saving'%cacheFile

272 273

274 - def getKey(self):

275 args = self.clientObj.arguments 276 key = args["pdb_ID"] + args["chain"] + args["ALGORITHM"] + \ 277 args["ITERATIONS"] + args["ESCORE"] + args["MAX_NUM_HOMOL"] 278 return string.strip(key)

279 280

281 - def checkForCompletion(self, page):

282 # opens the result page and check every 30 seconds if the 283 # computation is done by checking is the page contains 284 # "ConSurf finished calculation" 285 clientObj = self.clientObj 286 if self.flashNodesWhenRun is None: 287 self.flashNodesWhenRun = self.editor.flashNodesWhenRun 288 self.editor.flashNodesWhenRun = 0 # to prevent from turning gray 289 url = clientObj.urlBase + page + 'output.html' 290 print 'checkForCompletion', url 291 f = urllib.urlopen(url) 292 result = f.read() 293 f.close() 294 print 'checkForCompletion' 295 done = string.find(result, "ConSurf finished calculation") 296 if done != -1: 297 resultUrl = clientObj.urlBase + page + \ 298 'pdb%s.gradesPE'%clientObj.arguments['pdb_ID'] 299 key = self.getKey() 300 self.cache[key] = resultUrl 301 self.saveCache(self.cacheFile) 302 scores = clientObj.getConservationScores(resultUrl) 303 304 self.outputData(Fscore=map(lambda x: x[2], scores)) 305 self.outputData(Iscore=map(lambda x: x[3], scores)) 306 self.outputData(resultPageURL=resultUrl) 307 self.editor.flashNodesWhenRun = self.flashNodesWhenRun 308 if self.editor.flashNodesWhenRun: 309 col = self.getColor() 310 if col=='#557700': # after successfull run of failed node 311 col = node.colorBeforeFail 312 #else: 313 # self.dirty = 0 314 self.setColor('gray85') 315 316 # FIXME: should really schedule all children here 317 self.schedule() 318 return 319 320 if string.find(result, "ERROR") != -1: 321 print "ERROR ********************************************" 322 print result 323 return 324 #if count == 40: 325 # print "giving up after %d minutes"%(count*0.5,) 326 # break 327 self.editor.root.after( 30000, self.checkForCompletion, page )

328 329

330 - def __init__(self, name='Consurf Client', cacheFile='.consrf.cache', 331 **kw):

332 kw['name'] = name 333 self.runInBackground = 1 334 self.cacheFile = cacheFile 335 self.cache = self.readCache(cacheFile) 336 apply( NetworkNode.__init__, (self,), kw ) 337 338 self.inputPortsDescr.append( {'name':'molecule', 339 'datatype':'MoleculeSet'} ) 340 self.inputPortsDescr.append( {'name':'chain', 'required':False, 341 'datatype':'string'} ) 342 self.inputPortsDescr.append( {'name':'algorithm', 'required':False, 343 'datatype':'string'} ) 344 345 self.outputPortsDescr.append( {'name':'Fscore', 346 'datatype':'list'} ) 347 self.outputPortsDescr.append( {'name':'Iscore', 348 'datatype':'list'} ) 349 self.outputPortsDescr.append( {'name':'resultPageURL', 350 'datatype':'string'} ) 351 352 self.clientObj = ConsurfClient() 353 354 code = """def doit(self, molecule, chain, algorithm): 355 self.flashNodesWhenRun = None 356 mol = molecule[0] 357 assert algorithm is None or algorithm[0] in ['Likelihood', 'Parsimony'] 358 if chain is None or chain[0] not in mol.chains.id: 359 chain = molecule[0].chains[0].id 360 if algorithm is None: 361 algorithm = ['Likelihood'] 362 self.clientObj.setArguments( pdb_ID=mol.name, 363 chain=chain[0], ALGORITHM=algorithm[0]) 364 key = self.getKey() 365 if self.cache.has_key(key): 366 resultUrl = self.cache[key] 367 scores = self.clientObj.getConservationScores(resultUrl) 368 369 self.outputData(Fscore=map(lambda x: x[2], scores)) 370 self.outputData(Iscore=map(lambda x: x[3], scores)) 371 self.outputData(resultPageURL=resultUrl) 372 else: 373 result = self.clientObj.run() 374 resultPage = self.clientObj.getResultPage(result) 375 self.checkForCompletion(resultPage)\n""" 376 377 self.setFunction(code)

378 379

380 - def beforeAddingToNetwork(self, net):

381 try: 382 from MolKit.VisionInterface.MolKitNodes import molkitlib 383 net.editor.addLibraryInstance(molkitlib, 384 'MolKit.VisionInterface.MolKitNodes', 385 'molkitlib') 386 except: 387 print 'Warning! Could not import molkitlib from MolKit/VisionInterface'

388 389

390 -class GoogleClient(PublicHttpClient):

391

392 - def __init__(self):

393 self.urlBase = 'http://www.google.com/' 394 self.cgi = 'search' 395 url = self.urlBase + self.cgi 396 PublicHttpClient.__init__(self, url) 397 398 self.arguments = { 399 "q": '', 400 "hl": 'en', 401 "ie": 'ISO-8859-1', 402 }

403 404

405 -class GoogleClientNode(NetworkNode):

406

407 - def __init__(self, name='Consurf Client', cacheFile='.consrf.cache', 408 **kw):

409 kw['name'] = name 410 411 apply( NetworkNode.__init__, (self,), kw ) 412 self.widgetDescr['searchString'] = { 413 'class':'NEEntry', 'master':'node' 414 } 415 self.inputPortsDescr.append( { 416 'name':'searchString', 'datatype':'string'} ) 417 self.outputPortsDescr.append( { 418 'name':'html', 'datatype':'string' 419 } ) 420 self.outputPortsDescr.append( { 421 'name':'links', 'datatype':'list' 422 } ) 423 424 self.clientObj = GoogleClient() 425 426 code = """def doit(self, searchString): 427 428 self.outputData(html=resultHtml)\n""" 429 430 self.setFunction(code)

431 432

433 -class CGIFormNode(NetworkNode):

434

435 - def __init__(self, name='CGIForm Node', CGIFormObj=None, **kw):

436 kw['name'] = name 437 438 self.CGIFormObj = CGIFormObj 439 440 apply( NetworkNode.__init__, (self,), kw ) 441 442 if CGIFormObj is not None: 443 buildCGIWidget(self, CGIFormObj) 444 445 code = """def doit(self, *args): 446 argsDict = {} 447 for port, val in map(None, self.inputPorts, args): 448 argsDict[port.name] = val 449 apply(self.CGIFormObj.setArguments, (), argsDict) 450 result=self.CGIFormObj.run() 451 if result: 452 self.outputData(html=[result])\n""" 453 454 self.outputPortsDescr.append( { 455 'name':'html', 'datatype':'list' 456 } ) 457 458 self.setFunction(code)

459 460

461 - def getNodeDefinitionSourceCode(self, lib, nodeId, nodeUniqId, networkName, 462 ):

463 464 txt = '#create CGI Form object:\n' 465 txt = txt + 'from mglutil.web.HTMLParser import CGIForm\n' 466 txt = txt + 'obj='+self.CGIFormObj.getCreationSourceCode() 467 468 self.constrkw['CGIFormObj']='obj' # temporarily add kw to generate 469 # correct list of arguments 470 src = NetworkNode.getNodeDefinitionSourceCode(self, lib, nodeId, 471 nodeUniqId, networkName) 472 473 del self.constrkw['CGIFormObj'] # now we remove it 474 475 for line in src[0]: 476 txt = txt + line 477 478 return [txt], nodeId, nodeUniqId

479 480

481 -class GetCGIForms(NetworkNode):

482

483 - def __init__(self, name='GetCGIForms', **kw):

484 kw['name'] = name 485 486 apply( NetworkNode.__init__, (self,), kw ) 487 488 self.widgetDescr['url'] = {'class':'NEEntry', 'master':'node', 489 'labelCfg':{'text':'url:'}, 'labelSide':'left' } 490 self.inputPortsDescr.append( {'name':'url', 'datatype':'string'} ) 491 492 self.outputPortsDescr.append( {'name':'forms', 'datatype':'list'} ) 493 494 self.parserObj = ParseHTML(mode='forms') 495 496 code = """def doit(self, url): 497 498 if url is None or url == '': return 499 if(url[:7]) != 'http://': url = 'http://' + url 500 501 forms = self.parserObj.doit(url) 502 posx = 60 503 posy = 60 504 505 inputString = "" 506 argsString = "" 507 508 for form in forms: 509 node = CGIFormNode(CGIFormObj=None, name=form.name) 510 511 posX = self.posx + posx 512 posY = self.posy + posy 513 posx = posx + 20 514 posy = posy + 25 515 516 # this method is defined outside this node 517 buildCGIWidget(node, form) 518 519 self.editor.currentNetwork.addNode(node, posX, posY) 520 #node.addOutputPort(name='html', datatype='list') 521 node.CGIFormObj = form 522 node.CGIFormObj.setURL(url) 523 524 self.outputData(forms=forms)\n""" 525 526 self.setFunction(code)

527 528 529 from Vision.VPE import NodeLibrary 530 weblib = NodeLibrary('Web', '#bb4040') 531 532 weblib.addNode(ConsurfClientNode, 'Consurf', 'Filter') 533 #weblib.addNode(WebBrowser, 'Web Browser', 'Output') # not supported any longr 534 weblib.addNode(GetCGIForms, 'Get CGI Forms', 'Filter') 535 #weblib.addNode(CGIFormNode, 'CGI Form Node', 'Output') 536 537 538 if __name__ == '__main__': 539 print 'Creating client Object' 540 server = ConsurfServer() 541 print 'submitting job' 542 server.setArguments( pdb_ID='2plv', chain='1', ALGORITHM="Likelihood") 543 result = server.run() 544 print 'get temporary result page' 545 resultPage = server.getResultPage(result) 546 print ' wait for completion' 547 resultUrl = server.checkForCompletion(resultPage) 548 print ' COMPLETED' 549 scores = server.getConservationScores(resultUrl) 550 print scores 551 552 #c = GoogleClient() 553 #c.setArguments(q='foo') 554 #result = c.run() 555

Source Code for Module Vision.WebNodes