Biskit.LocalPath

41 """ 42 Encapsulate a file name that might look differently in different 43 environments depending on environment settings. The file name has 44 an absolute (original) variant but parts of it can be looked up 45 from environment or Biskit settings variables (if the original path 46 doesn't exist in the current environment). 47 48 Creating a LocalPath: 49 50 Creating a LocalPath is simple. By default, LocalPath takes a 51 normal filename and browses the local Biskit.settings and run-time 52 environment for variables that can replace part of that path. 53 54 Variable values must be at least 3 char long and contain at least 55 one '/' in order to avoid substitution of single letters or '/a' 56 etc. $PYTHONPATH, $PATH and $PWD are ignored. 57 58 Variables from Biskit.settings (thus also .biskit/settings.cfg) 59 have priority over normal environment variables both during the 60 construction of LocalPath instances and during the reconstruction 61 of a locally valid path. 62 63 Using a LocalPath: 64 65 LocalPath tries behaving like the simple absolute path string when it 66 comes to slicing etc:: 67 l = LocalPath( '{/home/raik|$USER}/data/x.txt' ) 68 l[:] == '/home/raik/data/x.txt' == l.local() 69 l[-4:] == '.txt' 70 str( l ) == l.local() 71 72 l.formatted() == '{/home/raik|$USER}/data/x.txt' 73 74 @todo: input: allow multiple or overlapping substitutions 75 output: only substitute what is necessary until path exists 76 """ 77 78 ## pattern for formatted string input 79 ex_fragment = re.compile('\{([a-zA-Z0-9_~/ ]+)\|\$([a-zA-Z0-9_]+)\}') 80 81 ## pattern for minimal path fragment that can be substituted 82 ex_minpath = re.compile( '/|~[a-zA-Z0-9_~/ ]+' ) 83 84 ## never use these variables 85 exclude_vars = ['PWD','OLDPWD','PYTHONPATH','PATH'] 86

87 - def __init__( self, path=None, checkEnv=1, minLen=3, maxSub=1, 88 absolute=1, resolveLinks=0, **vars ):

89 """ 90 Create a new environment-dependent path from either a list of 91 fragments and their substitution variable names or from a path or 92 from a formatted string (not implemented). 93 A path will be analyzed to substitute as big chunks as possible 94 by environment variables. ~ and ../../ will be expanded both in 95 the given path and in the environment variables. 96 97 @param path: path(s) 98 @type path: [ (str, str) ] OR str 99 @param checkEnv: look for substitution values among environment 100 variables (default 1) 101 @type checkEnv: 1|0 102 @param absolute: normalize file name [1] 103 @type absolute: 1|0 104 @param resolveLinks: resolve symbolic links [0] 105 @type resolveLinks: 1|0 106 @param maxSub: maximal number of substitutions [1] 107 @type maxSub: int 108 @param vars: additional param=value pairs with suggested substitutors 109 @type vars: param=value 110 """ 111 112 self.fragments = [] ## list of tuples (absolut,variable_name) 113 if path: 114 self.set( path, checkEnv=checkEnv, minLen=minLen, 115 absolute=absolute, resolveLinks=resolveLinks, 116 maxSub=maxSub, **vars ) 117 118 self.__hash = None 119 self.__cache = None

120 121

122 - def __setstate__(self, state ):

123 """ 124 called for unpickling the object. 125 """ 126 self.__dict__ = state 127 ## backwards compability 128 self.__hash = getattr( self, '_LocalPath__hash', None ) 129 self.__cache = None

130 131

132 - def get_local( self, existing=0 ):

133 """ 134 Return a valid, absolute path. Either the existing original or with 135 all substitutions for which environment variables exist. 136 This function is time consuming (absfile - os.realpath is the culprit). 137 138 @param existing: don't return a non-existing path 139 @type existing: 0|1 140 141 @return: valid absolute path in current environment 142 @rtype: str 143 144 @raise LocalPathError: if existing==1 and no existing path can be 145 constructed via environment variables 146 """ 147 result = string.join( [ f[0] for f in self.fragments ], '' ) 148 result = T.absfile( result ) 149 150 if os.path.exists( result ): 151 return result 152 153 substitutions = self.get_substitution_dict() 154 result = '' 155 for abs, env in self.fragments: 156 if env: 157 result += substitutions.get( env, abs ) 158 else: 159 result += abs 160 161 result = T.absfile( result ) 162 163 if existing and not os.path.exists( result ): 164 raise LocalPathError, "Can't construct existing path from %s."%\ 165 self.formatted() 166 167 return result

168 169

170 - def local( self, existing=0, force=1 ):

171 """ 172 Cached variant of get_local. 173 Return a valid, absolute path. Either the existing original or with 174 all substitutions for which environment variables exist. 175 This function is time consuming (absfile - os.realpath is the culprit). 176 177 @param existing: don't return a non-existing path [0] 178 @type existing: 0|1 179 180 @param force: override cached value [0] 181 @type force: 0|1 182 183 @return: valid absolute (not necessarily existing) path in current 184 environment 185 @rtype: str 186 187 @raise LocalPathError: if existing==1 and no existing path can be 188 constructed via environment variables 189 """ 190 191 if not existing and self.__cache and not force: 192 return self.__cache 193 194 r = self.get_local( existing=existing ) 195 196 if not existing: 197 self.__cache = r 198 199 return r

200 201

202 - def formatted( self ):

203 """ 204 Get a string representation that describes the original path and all 205 possible substitutions by environment variables. 206 207 @return: formated path e.g. C{ '{/home/raik|$USER}/data/x.txt' } 208 @rtype: str 209 """ 210 r = "" 211 for absolut,var in self.fragments: 212 if var: 213 r += "{%s|$%s}" % (absolut, var) 214 else: 215 r += absolut 216 return r

217 218

219 - def original( self ):

220 """ 221 Get the original path (also non-absolute) that is used if the file 222 exists or if there are no environment variables for any of the 223 substitutions. 224 225 @return: original path 226 @rtype: str 227 """ 228 result = [ f[0] for f in self.fragments ] 229 return string.join( result, '' )

230 231

232 - def set( self, v, checkEnv=1, minLen=3, maxSub=1, 233 absolute=1, resolveLinks=0, **vars ):

234 """ 235 Assign a new file name. checkEnv, minLen, resolve*, maxSub are 236 only considered for path name input. 237 238 @param v: fragment tuples or path or custom-formatted string 239 @type v: [ (str,str) ] OR str 240 @param checkEnv: look for possible substitutions in environment [1] 241 (iggnored if v is already formatted like '{/x/y|$xy}/z.txt' ) 242 @type checkEnv: 0|1 243 @param minLen: mininal length of environment variables to consider [3] 244 @type minLen: int 245 @param absolute: normalize file name [1] 246 @type absolute: 1|0 247 @param resolveLinks: resolve symbolic links [0] 248 @type resolveLinks: 1|0 249 @param maxSub: maximal number of substitutions [1] 250 @type maxSub: int 251 @param vars: additional param=value pairs with suggested substitutors 252 @type vars: param=value 253 """ 254 if type( v ) == list: 255 return self.set_fragments( v ) 256 257 if type( v ) == str and '{' in v: 258 return self.set_string( v ) 259 260 if type( v ) == str and (checkEnv or vars): 261 return self.set_path( v, minLen=minLen, absolute=absolute, 262 maxSub=maxSub, 263 resolveLinks=resolveLinks, **vars ) 264 265 raise LocalPathError, 'incompatible value for LocalPath' + str(v) 266 267 self.__hash = None 268 self.__cache = None

269 270

271 - def set_fragments( self, *fragments ):

272 """ 273 Set new path from list of path fragments and their possible 274 environment variable substitutions. Fragments that can not be 275 substituted are given as (str, None). 276 277 @param fragments: list of fragment tuples 278 @type fragments: [ (str, str), (str, None), .. ] 279 """ 280 self.fragments = fragments 281 self.__hash = None 282 self.__cache = None

283 284

285 - def set_string( self, s ):

286 """ 287 Set a new path and its substitutable parts from a formatted string. 288 289 @param s: formatted like {/home/raik|$USER}/data/test.txt 290 @type s: str 291 292 @raise PathError: formatted input is not yet implemented 293 """ 294 fragments = [] 295 pos = 0 296 297 for m in self.ex_fragment.finditer( s ): 298 299 if m.start() > pos: 300 fragments += [ ( s[pos:m.start()], None ) ] 301 302 fragments += [ m.groups() ] 303 304 pos = m.end() 305 306 if len( s ) > pos: 307 fragments += [ ( s[pos:len(s)], None ) ] 308 309 self.set_fragments( *fragments )

310 311

312 - def absfile( self, fname, resolveLinks=0 ):

313 return T.absfile( fname, resolveLinks=resolveLinks )

314 315

316 - def set_path( self, fname, minLen=3, absolute=1, resolveLinks=0, 317 maxSub=1, **vars ):

318 """ 319 Set a new path and try to identify settings/environment 320 variables that could substitute parts of it. 321 322 @param fname: relative or absolute file name 323 @type fname: str 324 @param minLen: minimal length of string o to be counted as path 325 @type minLen: int 326 @param absolute: normalize file name [1] 327 @type absolute: 1|0 328 @param resolveLinks: resolve symbolic links [0] 329 @type resolveLinks: 1|0 330 @param maxSub: maximal number of substitutions [1] 331 @type maxSub: int 332 @param vars: additional param=value pairs with suggested substitutors 333 @type vars: param=value 334 """ 335 env_items = self.get_substitution_pairs( minLen=minLen, vars=vars ) 336 337 if absolute: 338 fname = self.absfile( fname, resolveLinks=resolveLinks ) 339 340 fragments = [ ( fname, None ) ] ## default result 341 342 substitutions=0 343 344 for name, value in env_items: 345 old = fragments 346 fragments = self.__substitute( fragments, name, value ) 347 348 substitutions += (old != fragments) 349 if substitutions == maxSub: 350 break 351 352 self.fragments = fragments 353 354 self.__hash = None 355 self.__cache = None

356 357

358 - def exists( self ):

359 """ 360 Check if path exists 361 362 @return: 1 if if current path exists 363 @rtype: 1|0 364 """ 365 return os.path.exists( self.local() )

366 367

368 - def load( self ):

369 """ 370 Try to unpickle an object from the currently valid path. 371 372 @return: unpickled object 373 @rtype: any 374 375 @raise IOError: if file can not be found 376 """ 377 try: 378 return T.Load( self.local( existing=1 ) ) 379 except LocalPathError, why: 380 raise IOError, "Cannot find file %s (constructed from %s)" %\ 381 self.local(), str( self )

382 383

384 - def dump( self, o ):

385 """ 386 Try to pickle an object to the currently valid path. 387 388 @return: the absolute path to which o was pickled 389 @rtype: str 390 """ 391 try: 392 f = self.local() 393 T.Dump( f, o ) 394 return f 395 except: 396 T.errWriteln("Couldn't dump to %s (constructed from %s)" %\ 397 self.formatted(), self.local() ) 398 raise

399

400 - def __find_subpath( self, path, subpath ):

401 """ 402 403 """ 404 seps = [ i for i in range( len(path) ) if path[i]==os.path.sep ] 405 seps += [ len( path ) ] 406 407 pos = path.find( subpath ) 408 409 if pos in seps and pos+len(subpath) in seps: 410 return pos 411 412 return -1

413 414

415 - def __substitute( self, fragments, name, value ):

416 """ 417 Look in all not yet substituted fragments for parts that can be 418 substituted by value and, if successful, create a new fragment 419 420 @param fragments: fragment tuples 421 @type fragments: [ (str, str) ] 422 @param name: substitution variable name 423 @type name: str 424 @param value: susbtitution value in current environment 425 @type value: str 426 427 @return: fragment tuples 428 @rtype: [ (str, str) ] 429 """ 430 result = [] 431 432 try: 433 for abs, subst in fragments: 434 435 if not subst: ## unsubstituted fragment 436 437 ## pos = abs.find( value ) 438 pos = self.__find_subpath( abs, value ) 439 440 if pos != -1: 441 end = pos + len( value ) 442 443 f1, f2, f3 = abs[0:pos], abs[pos:end], abs[end:] 444 445 if f1: 446 result += [ (f1, None) ] ## unsubstituted head 447 result += [ (f2, name) ] ## new substitution 448 if f3: 449 result += [ (f3, None) ] ## unsubstituted tail 450 451 else: 452 result += [ (abs, subst) ] 453 else: 454 result += [ (abs, subst ) ] 455 except OSError, why: 456 EHandler.fatal("Substituting path fragments: \n" + 457 str( fragments ) + '\nname: ' + str( name ) + 458 '\nvalue:' + str( value ) ) 459 460 return result

461 462

463 - def __is_path( self, o, minLen=3 ):

464 """ 465 Check whether an object is a path string (existing or not). 466 467 @param minLen: minimal length of string o to be counted as path 468 @type minLen: int 469 470 @return: 1|0 471 @rtype: int 472 """ 473 r = ( type( o ) == str and o.find('/') != -1 and len(o) >= minLen\ 474 and o.find(':') == -1 ) 475 if r: 476 try: 477 s = T.absfile( o ) 478 return 1 479 except: 480 return 0 481 return 0

482 483

484 - def __path_vars( self, d, minLen=3, vars={}, exclude=[] ):

485 """ 486 @see L{__paths_in_settings} and L{__paths_in_env} 487 488 @return: [ (variable name, path) ] sorted by length of value 489 @rtype: [ (str,str) ] 490 """ 491 492 items = vars.items() + d.items() 493 exclude = exclude 494 495 items = [ (k,v) for (k,v) in items if self.__is_path(v) ] 496 497 pairs = [ (len(v[1]), v) for v in items 498 if not v[0] in self.exclude_vars ] 499 500 pairs.sort() 501 pairs.reverse() 502 503 return [ x[1] for x in pairs ]

504 505

506 - def __paths_in_settings( self, minLen=3, vars={}, exclude=[]):

507 """ 508 Get all setting variables looking like a path, sorted by length 509 510 @param minLen: minimal path length [3] 511 @type minLen: int 512 @param vars: alternative param=value pairs to consider 513 instead of environment 514 @type vars: param=value 515 516 @return: [ (variable name, value) ] sorted by length of value 517 @rtype: [ (str,str) ] 518 """ 519 return self.__path_vars( S.__dict__, minLen=minLen, vars=vars, 520 exclude=(exclude + self.exclude_vars ) )

521 522

523 - def __paths_in_env( self, minLen=3, vars={}, exclude=[] ):

524 """ 525 Get all environment variables with at least one '/' sorted by length. 526 527 @param minLen: minimal path length [3] 528 @type minLen: int 529 @param vars: alternative param=value pairs to consider 530 instead of environment 531 @type vars: param=value 532 533 @return: [ (variable name, value) ] sorted by length of value 534 @rtype: [ (str,str) ] 535 """ 536 return self.__path_vars( os.environ, minLen=minLen, vars=vars, 537 exclude=(exclude + self.exclude_vars ) )

538 539

540 - def get_substitution_pairs( self, minLen=3, vars={}, exclude=[] ):

541 """ 542 Get all variable/value pairs that are available for path substitutions. 543 544 @param minLen: minimal path length [3] 545 @type minLen: int 546 @param vars: additional param=value pairs to consider 547 @type vars: param=value 548 549 @return: [ (variable name, value) ] sorted by priority 550 (mostly length of value) 551 @rtype: [ (str,str) ] 552 """ 553 r = self.__paths_in_settings(minLen=minLen,vars=vars, exclude=exclude ) 554 r +=self.__paths_in_env( minLen=minLen,vars=vars, exclude=exclude ) 555 return r

556 557

558 - def get_substitution_dict( self, minLen=3, vars={}, exclude=[] ):

559 return dict(self.get_substitution_pairs(minLen=minLen,vars=vars, 560 exclude=exclude))

561 562

563 - def __str__( self ):

564 """ 565 @return: Same as local(). string representation for print and str() 566 @rtype: str 567 """ 568 return self.local()

569 570

571 - def __repr__( self ):

572 """ 573 @return: formatted output (Python representation) 574 @rtype: str 575 """ 576 return "LocalPath[ %s ]" % self.formatted()

577 578

579 - def __len__( self ):

580 """ 581 Time costly when repeated many times. 582 583 @return: length of file name in current environment 584 @rtype: int 585 """ 586 return len( self.local() )

587 588

589 - def __getslice__( self, a, b ):

590 return self.local()[a:b]

591 592

593 - def __getitem__( self, i ):

594 return self.local()[i]

595 596

597 - def __eq__( self, other ):

598 """ 599 supports this == other -> 0|1 600 """ 601 if not isinstance( other, LocalPath ): 602 return 0 603 return self.fragments == other.fragments

604 605

606 - def __ne__( self, other ):

607 """ 608 supports this != other -> 0|1 609 """ 610 if not isinstance( other, LocalPath ): 611 return 1 612 return self.fragments != other.fragments

613 614

615 - def __hash__( self ):

616 """ 617 if __eq__ or __cmp__ are defined hash has to be defined too, otherwise 618 the objects cannot be used as keys in dictionaries (needed for Complex- 619 ModelRegistry). 620 621 @return: int 622 @rtype: 623 """ 624 if self.__hash is None: 625 self.__hash = self.formatted().__hash__() 626 627 return self.__hash

Source Code for Module Biskit.LocalPath