1 #!/usr/bin/python2.4 2 # 3 # CDDL HEADER START 4 # 5 # The contents of this file are subject to the terms of the 6 # Common Development and Distribution License (the "License"). 7 # You may not use this file except in compliance with the License. 8 # 9 # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 # or http://www.opensolaris.org/os/licensing. 11 # See the License for the specific language governing permissions 12 # and limitations under the License. 13 # 14 # When distributing Covered Code, include this CDDL HEADER in each 15 # file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 # If applicable, add the following below this CDDL HEADER, with the 17 # fields enclosed by brackets "[]" replaced with your own identifying 18 # information: Portions Copyright [yyyy] [name of copyright owner] 19 # 20 # CDDL HEADER END 21 # 22 # Copyright 2009 Sun Microsystems, Inc. All rights reserved. 23 # Use is subject to license terms. 24 25 26 import fnmatch 27 import getopt 28 import gettext 29 import os 30 import pkg.depotcontroller as depotcontroller 31 import pkg.publish.transaction as trans 32 import re 33 import shlex 34 import sys 35 import urllib 36 import urlparse 37 38 from datetime import datetime 39 from itertools import groupby 40 from pkg import actions, elf 41 from pkg.bundle.SolarisPackageDirBundle import SolarisPackageDirBundle 42 from pkg.sysvpkg import SolarisPackage 43 from tempfile import mkstemp 44 45 gettext.install("import", "/usr/lib/locale") 46 47 class package(object): 48 def __init__(self, name): 49 self.name = name 50 self.files = [] 51 self.depend = [] 52 self.file_depend = [] 53 self.idepend = [] #svr4 pkg deps, if any 54 self.undepend = [] 55 self.extra = [] 56 self.dropped_licenses = [] 57 self.nonhollow_dirs = {} 58 self.srcpkgs = [] 59 self.classification = "" 60 self.desc = "" 61 self.version = "" 62 self.imppkg = None 63 pkgdict[name] = self 64 65 def import_pkg(self, imppkg, line): 66 try: 67 p = SolarisPackage(pkg_path(imppkg)) 68 except: 69 raise RuntimeError("No such package: '%s'" % imppkg) 70 71 self.imppkg = p 72 73 svr4pkgpaths[p.pkginfo["PKG.PLAT"]] = pkg_path(imppkg) 74 75 # filename NOT always same as pkgname 76 imppkg = p.pkginfo["PKG.PLAT"] 77 svr4pkgsseen[imppkg] = p 78 79 if "SUNW_PKG_HOLLOW" in p.pkginfo and \ 80 p.pkginfo["SUNW_PKG_HOLLOW"].lower() == "true": 81 hollow_pkgs[imppkg] = True 82 83 excludes = dict((f, True) for f in line.split()) 84 85 # XXX This isn't thread-safe. We want a dict method that adds 86 # the key/value pair, but throws an exception if the key is 87 # already present. 88 for o in p.manifest: 89 if o.pathname in excludes: 90 print "excluding %s from %s" % \ 91 (o.pathname, imppkg) 92 continue 93 94 if o.pathname in elided_files: 95 print "ignoring %s in %s" % (o.pathname, imppkg) 96 continue 97 98 if o.type == "e": 99 if o.pathname not in editable_files: 100 editable_files[o.pathname] = \ 101 [(imppkg, self)] 102 else: 103 editable_files[o.pathname].append( 104 (imppkg, self)) 105 106 # XXX This decidedly ignores "e"-type files. 107 108 if o.type in "fv" and o.pathname in usedlist: 109 s = reuse_err % ( 110 o.pathname, 111 self.name, 112 imppkg, 113 svr4pkgpaths[imppkg], 114 usedlist[o.pathname][1].name, 115 usedlist[o.pathname][0], 116 svr4pkgpaths[usedlist[o.pathname][0]]) 117 print s 118 raise RuntimeError(s) 119 elif o.type == "i" and o.pathname == "copyright": 120 # Fake up a unique path for each license. 121 o.pathname = "//license/%s" % imppkg 122 usedlist[o.pathname] = (imppkg, self) 123 self.files.append(o) 124 elif o.type != "i": 125 if o.type in "dx" and imppkg not in hollow_pkgs: 126 self.nonhollow_dirs[o.pathname] = True 127 128 usedlist[o.pathname] = (imppkg, self) 129 self.check_perms(o) 130 self.files.append(o) 131 132 if not self.version: 133 self.version = "%s-%s" % (def_vers, 134 get_branch(self.name)) 135 if not self.desc: 136 self.desc = zap_strings(p.pkginfo["NAME"], 137 description_detritus) 138 139 # This is how we'd import dependencies, but we'll use 140 # file-specific dependencies only, since these tend to be 141 # broken. 142 # self.depend.extend( 143 # d.req_pkg_fmri 144 # for d in p.deps 145 # ) 146 147 self.add_svr4_src(imppkg) 148 149 def add_svr4_src(self, imppkg): 150 if imppkg in destpkgs: 151 destpkgs[imppkg].append(self.name) 152 else: 153 destpkgs[imppkg] = [self.name] 154 self.srcpkgs.append(imppkg) 155 156 def import_file(self, fname, line): 157 imppkgname = self.imppkg.pkginfo["PKG.PLAT"] 158 159 if "SUNW_PKG_HOLLOW" in self.imppkg.pkginfo and \ 160 self.imppkg.pkginfo["SUNW_PKG_HOLLOW"].lower() == "true": 161 hollow_pkgs[imppkgname] = True 162 163 if fname in usedlist: 164 t = [ 165 f for f in usedlist[fname][1].files 166 if f.pathname == fname 167 ][0].type 168 if t in "fv": 169 assert imppkgname == usedlist[fname][0] 170 raise RuntimeError(reuse_err % ( 171 fname, 172 self.name, 173 self.imppkg, 174 svr4pkgpaths[self.imppkg], 175 usedlist[fname][1].name, 176 usedlist[fname][0], 177 svr4pkgpaths[usedlist[fname][0]])) 178 179 usedlist[fname] = (imppkgname, self) 180 o = [ 181 o 182 for o in self.imppkg.manifest 183 if o.pathname == fname 184 ] 185 # There should be only one file with a given pathname in a 186 # single package. 187 if len(o) != 1: 188 print "ERROR: %s %s" % (imppkgname, fname) 189 assert len(o) == 1 190 191 if line: 192 a = actions.fromstr( 193 "%s path=%s %s" % \ 194 ( 195 self.convert_type(o[0].type), 196 o[0].pathname, 197 line 198 ) 199 ) 200 for attr in a.attrs: 201 if attr == "owner": 202 o[0].owner = a.attrs[attr] 203 elif attr == "group": 204 o[0].group = a.attrs[attr] 205 elif attr == "mode": 206 o[0].mode = a.attrs[attr] 207 self.check_perms(o[0]) 208 self.files.extend(o) 209 210 def convert_type(self, svrtype): 211 """ given sv4r type, return IPS type""" 212 return { 213 "f": "file", "e": "file", "v": "file", 214 "d": "dir", "x": "dir", 215 "s": "link", 216 "l": "hardlink" 217 }[svrtype] 218 219 def type_convert(self, ipstype): 220 """ given IPS type, return svr4 type(s)""" 221 return { 222 "file": "fev", "dir": "dx", "link": "s", 223 "hardlink": "l" 224 }[ipstype] 225 226 def file_to_action(self, f): 227 228 if f.type in "dx": 229 action = actions.directory.DirectoryAction( 230 None, mode = f.mode, owner = f.owner, 231 group = f.group, path = f.pathname) 232 elif f.type in "efv": 233 action = actions.file.FileAction( 234 None, mode = f.mode, owner = f.owner, 235 group = f.group, path = f.pathname) 236 elif f.type == "s": 237 action = actions.link.LinkAction(None, 238 target = f.target, path = f.pathname) 239 elif f.type == "l": 240 action = actions.hardlink.HardLinkAction(None, 241 target = f.target, path = f.pathname) 242 else: 243 print "unknown type %s - path %s" % \ 244 ( f.type, f.pathname) 245 246 return action 247 248 def check_perms(self, manifest): 249 if manifest.type not in "fevdxbc": 250 return 251 252 if manifest.owner == "?": 253 manifest.owner = "root" 254 print "File %s in pkg %s owned by '?': mapping to %s" \ 255 % (manifest.pathname, self.name, manifest.owner) 256 257 if manifest.group == "?": 258 manifest.group = "bin" 259 print "File %s in pkg %s of group '?': mapping to %s" \ 260 % (manifest.pathname, self.name, manifest.group) 261 if manifest.mode == "?": 262 manifest.mode = "0444" 263 print "File %s in pkg %s mode '?': mapping to %s" % \ 264 (manifest.pathname, self.name, manifest.mode) 265 266 267 def chattr(self, fname, line): 268 o = [f for f in self.files if f.pathname == fname] 269 if not o: 270 raise RuntimeError("No file '%s' in package '%s'" % \ 271 (fname, curpkg.name)) 272 273 line = line.rstrip() 274 275 # is this a deletion? 276 if line.startswith("drop"): 277 for f in o: 278 # deletion of existing attribute 279 if not hasattr(f, "deleted_attrs"): 280 f.deleted_attrs = [] 281 print "Adding drop on %s of %s" % \ 282 (fname, line.split()[1:]) 283 f.deleted_attrs.extend(line.split()[1:]) 284 return 285 286 # handle insertion/modification case 287 for f in o: 288 # create attribute dictionary from line 289 new_type = self.convert_type(f.type) 290 new_attrs = actions._fromstr("%s %s" % 291 (new_type, line.rstrip()))[2] 292 # get path if we're not changing it 293 if "path" not in new_attrs: 294 new_attrs["path"] = fname 295 a = actions.types[new_type](**new_attrs) 296 if show_debug: 297 print "Updating attributes on " + \ 298 "'%s' in '%s' with '%s'" % \ 299 (f.pathname, curpkg.name, a) 300 orig_action = self.file_to_action(f) 301 302 if not hasattr(f, "changed_attrs"): 303 f.changed_attrs = {} 304 305 # each chattr produces a dictionary of actions 306 # including a path=xxxx and whatever modifications 307 # are made. Note that the path value may be a list in 308 # the case of modifications to the path... since 309 # each chattr produces another path= entry and results 310 # from applying the changes to the original file spec, 311 # we need to ignore path if it hasn't changed... for 312 # generality, we ignore all unchanged attributes in 313 # the code below, adding into changed_attrs only those 314 # that are different from the original... this also 315 # insulates us from the possibility of actions.fromstr 316 # adding additional attributes in the constructor... 317 318 for key in a.attrs.keys(): 319 if key not in orig_action.attrs or \ 320 orig_action.attrs[key] != a.attrs[key]: 321 if key in f.changed_attrs: 322 print "Warning: overwriting " \ 323 "changed attr %s on %s " \ 324 "from %s to %s" % \ 325 (key, f.pathname, 326 f.changed_attrs[key], 327 a.attrs[key]) 328 f.changed_attrs[key] = a.attrs[key] 329 330 331 # apply a chattr to wildcarded files/dirs 332 # also allows package specification, wildcarding, regexp edit 333 334 def chattr_glob(self, glob, line): 335 args = line.split() 336 if args[0] == "from": 337 args.pop(0) 338 pkgglob = args.pop(0) 339 line = " ".join(args) 340 else: 341 pkgglob = "*" 342 343 if args[0] == "type": # we care about type 344 args.pop(0) 345 types = self.type_convert(args.pop(0)) 346 line = " ".join(args) 347 else: 348 types = "dfevslx" 349 350 if args[0] == "edit": # we're doing regexp edit of attr 351 edit = True 352 args.pop(0) 353 target = args.pop(0) 354 regexp = re.compile(args.pop(0)) 355 replace = args.pop(0) 356 line = " ".join(args) 357 else: 358 edit = False 359 360 o = [ 361 f 362 for f in self.files 363 if fnmatch.fnmatchcase(f.pathname, glob) and 364 fnmatch.fnmatchcase( 365 usedlist[f.pathname][0], pkgglob) and 366 f.type in types 367 ] 368 369 chattr_line = line 370 371 for f in o: 372 fname = f.pathname 373 orig_action = self.file_to_action(f) 374 if edit: 375 if target in orig_action.attrs: 376 old_value = orig_action.attrs[target] 377 new_value = regexp.sub(replace, \ 378 old_value) 379 if old_value == new_value: 380 continue 381 chattr_line = "%s=%s %s" % \ 382 (target, new_value, line) 383 else: 384 continue 385 chattr_line = chattr_line.rstrip() 386 if show_debug: 387 print "Updating attributes on " + \ 388 "'%s' in '%s' with '%s'" % \ 389 (fname, curpkg.name, chattr_line) 390 391 # create attribute dictionary from line 392 new_type = self.convert_type(f.type) 393 new_attrs = actions._fromstr("%s %s" % 394 (new_type, chattr_line.rstrip()))[2] 395 # get path if we're not changing it 396 if "path" not in new_attrs: 397 new_attrs["path"] = fname 398 a = actions.types[new_type](**new_attrs) 399 # each chattr produces a dictionary of actions 400 # including a path=xxxx and whatever modifications 401 # are made. Note that the path value may be a list in 402 # the case of modifications to the path... since 403 # each chattr produces another path= entry and results 404 # from applying the changes to the original file spec, 405 # we need to ignore path if it hasn't changed... for 406 # generality, we ignore all unchanged attributes in 407 # the code below, adding into changed_attrs only those 408 # that are different from the original... this also 409 # insulates us from the possibility of actions.fromstr 410 # adding additional attributes in the constructor... 411 412 if not hasattr(f, "changed_attrs"): 413 f.changed_attrs = {} 414 for key in a.attrs.keys(): 415 if key not in orig_action.attrs or \ 416 orig_action.attrs[key] != a.attrs[key]: 417 if key in f.changed_attrs: 418 print "Warning: overwriting " \ 419 "changed attr %s on %s " \ 420 "from %s to %s" % \ 421 (key, f.pathname, 422 f.changed_attrs[key], 423 a.attrs[key]) 424 f.changed_attrs[key] = a.attrs[key] 425 426 pkgpaths = {} 427 428 def pkg_path(pkgname): 429 name = os.path.basename(pkgname) 430 if pkgname in pkgpaths: 431 return pkgpaths[name] 432 if "/" in pkgname: 433 pkgpaths[name] = os.path.realpath(pkgname) 434 return pkgname 435 else: 436 for each_path in wos_path: 437 if os.path.exists(each_path + "/" + pkgname): 438 pkgpaths[name] = each_path + "/" + pkgname 439 return pkgpaths[name] 440 441 raise RuntimeError("package %s not found" % pkgname) 442 443 444 def start_package(pkgname): 445 return package(pkgname) 446 447 def end_package(pkg): 448 pkg_branch = get_branch(pkg.name) 449 if not pkg.version: 450 pkg.version = "%s-%s" % (def_vers, pkg_branch) 451 elif "-" not in pkg.version: 452 pkg.version += "-%s" % pkg_branch 453 454 print "Package '%s'" % pkg.name 455 print " Version:", pkg.version 456 print " Description:", pkg.desc 457 print " Classification: ", pkg.classification 458 459 def publish_pkg(pkg): 460 461 new_pkg_name = "%s@%s" % (pkg.name, pkg.version) 462 t = trans.Transaction(def_repo, create_repo=create_repo, 463 pkg_name=new_pkg_name, noexecute=nopublish) 464 465 print " open %s" % new_pkg_name 466 transaction_id = t.open() 467 468 # Publish non-file objects first: they're easy. 469 for f in pkg.files: 470 if f.type in "dx": 471 action = actions.directory.DirectoryAction( 472 None, mode = f.mode, owner = f.owner, 473 group = f.group, path = f.pathname) 474 if hasattr(f, "changed_attrs"): 475 action.attrs.update(f.changed_attrs) 476 # chattr may have produced two path values 477 action.attrs["path"] = \ 478 action.attrlist("path")[-1] 479 print " %s add dir %s %s %s %s" % ( 480 pkg.name, 481 action.attrs["mode"], 482 action.attrs["owner"], 483 action.attrs["group"], 484 action.attrs["path"] 485 ) 486 elif f.type == "s": 487 action = actions.link.LinkAction(None, 488 target = f.target, path = f.pathname) 489 if hasattr(f, "changed_attrs"): 490 action.attrs.update(f.changed_attrs) 491 # chattr may have produced two path values 492 action.attrs["path"] = \ 493 action.attrlist("path")[-1] 494 print " %s add link %s %s" % ( 495 pkg.name, 496 action.attrs["path"], 497 action.attrs["target"] 498 ) 499 elif f.type == "l": 500 action = actions.hardlink.HardLinkAction(None, 501 target = f.target, path = f.pathname) 502 if hasattr(f, "changed_attrs"): 503 action.attrs.update(f.changed_attrs) 504 # chattr may have produced two path values 505 action.attrs["path"] = \ 506 action.attrlist("path")[-1] 507 pkg.depend += process_link_dependencies( 508 action.attrs["path"], action.attrs["target"]) 509 print " %s add hardlink %s %s" % ( 510 pkg.name, 511 action.attrs["path"], 512 action.attrs["target"] 513 ) 514 else: 515 continue 516 517 # 518 # If the originating package was hollow, tag this file 519 # as being global zone only. 520 # 521 522 if f.type not in "dx" and f.pathname in usedlist and \ 523 usedlist[f.pathname][0] in hollow_pkgs: 524 action.attrs["opensolaris.zone"] = "global" 525 action.attrs["variant.opensolaris.zone"] = "global" 526 527 if f.type in "dx" and f.pathname in usedlist and \ 528 usedlist[f.pathname][0] in hollow_pkgs and \ 529 f.pathname not in pkg.nonhollow_dirs: 530 action.attrs["opensolaris.zone"] = "global" 531 action.attrs["variant.opensolaris.zone"] = "global" 532 533 # handle attribute deletion 534 if hasattr(f, "deleted_attrs"): 535 for d in f.deleted_attrs: 536 if d in action.attrs: 537 del action.attrs[d] 538 539 t.add(action) 540 541 # Group the files in a (new) package based on what (old) package they 542 # came from, so that we can iterate through all files in a single (old) 543 # package (and, therefore, in a single bzip2 archive) before moving on 544 # to the next. Because groupby() needs its input pre-sorted by group 545 # and we want to maintain the order that the files come out of the cpio 546 # archives, we coalesce the groups with the groups dictionary. 547 def fn(key): 548 return usedlist[key.pathname][0] 549 groups = {} 550 for k, g in groupby((f for f in pkg.files if f.type in "fevi"), fn): 551 if k in groups: 552 groups[k].extend(g) 553 else: 554 groups[k] = list(g) 555 556 def otherattrs(action): 557 s = " ".join( 558 "%s=%s" % (a, action.attrs[a]) 559 for a in action.attrs 560 if a not in ("owner", "group", "mode", "path") 561 ) 562 if s: 563 return " " + s 564 else: 565 return "" 566 567 # Maps class names to preserve attribute values. 568 preserve_dict = { 569 "renameold": "renameold", 570 "renamenew": "renamenew", 571 "preserve": "true", 572 "svmpreserve": "true" 573 } 574 575 undeps = set() 576 for g in groups.values(): 577 pkgname = usedlist[g[0].pathname][0] 578 print "pulling files from archive in package", pkgname 579 bundle = SolarisPackageDirBundle(svr4pkgpaths[pkgname]) 580 pathdict = dict((f.pathname, f) for f in g) 581 for f in bundle: 582 if f.name == "license": 583 if f.attrs["license"] in pkg.dropped_licenses: 584 continue 585 # add transaction id so that every version 586 # of a pkg will have a unique license to prevent 587 # license from disappearing on upgrade 588 f.attrs["transaction_id"] = transaction_id 589 # The "path" attribute is confusing and 590 # unnecessary for licenses. 591 del f.attrs["path"] 592 print " %s add license %s" % \ 593 (pkg.name, f.attrs["license"]) 594 t.add(f) 595 elif f.attrs["path"] in pathdict: 596 if pkgname in hollow_pkgs: 597 f.attrs["opensolaris.zone"] = "global" 598 f.attrs["variant.opensolaris.zone"] = \ 599 "global" 600 path = f.attrs["path"] 601 if pathdict[path].type in "ev": 602 f.attrs["preserve"] = "true" 603 f.attrs["owner"] = pathdict[path].owner 604 f.attrs["group"] = pathdict[path].group 605 f.attrs["mode"] = pathdict[path].mode 606 607 # is this a file for which we need a timestamp? 608 basename = os.path.basename(path) 609 for file_pattern in timestamp_files: 610 if fnmatch.fnmatch(basename, 611 file_pattern): 612 break 613 else: 614 del f.attrs["timestamp"] 615 if pathdict[path].klass in preserve_dict.keys(): 616 f.attrs["preserve"] = \ 617 preserve_dict[pathdict[path].klass] 618 if hasattr(pathdict[path], "changed_attrs"): 619 f.attrs.update( 620 pathdict[path].changed_attrs) 621 # chattr may have produced two values 622 f.attrs["path"] = f.attrlist("path")[-1] 623 624 print " %s add file %s %s %s %s%s" % \ 625 (pkg.name, f.attrs["mode"], 626 f.attrs["owner"], f.attrs["group"], 627 f.attrs["path"], otherattrs(f)) 628 629 # handle attribute deletion 630 if hasattr(pathdict[path], "deleted_attrs"): 631 for d in pathdict[path].deleted_attrs: 632 if d in f.attrs: 633 print "removed %s from %s in pkg %s" % (d, path, new_pkg_name) 634 del f.attrs[d] 635 636 # Read the file in chunks to avoid a memory 637 # footprint blowout. 638 fo = f.data() 639 bufsz = 256 * 1024 640 sz = int(f.attrs["pkg.size"]) 641 fd, tmp = mkstemp(prefix="pkg.") 642 while sz > 0: 643 d = fo.read(min(bufsz, sz)) 644 os.write(fd, d) 645 sz -= len(d) 646 d = None 647 os.close(fd) 648 649 # Fool the action into pulling from a 650 # temporary file so that both add() and 651 # process_dependencies() can read() the 652 # data. 653 f.data = lambda: open(tmp, "rb") 654 t.add(f) 655 656 # Look for dependencies 657 deps, u = process_dependencies(tmp, path) 658 pkg.depend += deps 659 if u: 660 print \ 661 "%s has missing dependencies: %s" \ 662 % (path, u) 663 undeps |= set(u) 664 os.unlink(tmp) 665 666 # process any dependencies on files 667 for f in pkg.file_depend: 668 f = f.lstrip("/") # remove any leading / 669 if f in usedlist: 670 pkg.depend += [ "%s@%s" % 671 (usedlist[f][1].name, 672 usedlist[f][1].version) 673 ] 674 else: 675 print "Warning: pkg %s: depend_path %s not satisfied" \ 676 % (pkg.name, f) 677 undeps.add(f) 678 # Publish dependencies 679 680 missing_cnt = 0 681 682 for p in set(pkg.idepend): # over set of svr4 deps, append ipkgs 683 if p in destpkgs: 684 pkg.depend.extend(destpkgs[p]) 685 else: 686 print "pkg %s: SVR4 package %s not seen" % \ 687 (pkg.name, p) 688 missing_cnt += 1 689 if missing_cnt > 0: 690 raise RuntimeError("missing packages!") 691 692 for p in set(pkg.depend) - set(pkg.undepend): 693 # Don't make a package depend on itself. 694 if p.split("@")[0] == pkg.name: 695 continue 696 # enhance unqualified dependencies to include current 697 # pkg version 698 if "@" not in p and p in pkgdict: 699 p = "%s@%s" % (p, pkgdict[p].version) 700 701 print " %s add depend require %s" % (pkg.name, p) 702 action = actions.depend.DependencyAction(None, 703 type = "require", fmri = p) 704 t.add(action) 705 706 for a in pkg.extra: 707 print " %s add %s" % (pkg.name, a) 708 action = actions.fromstr(a) 709 if hasattr(action, "hash"): 710 fname, fd = sourcehook(action.hash) 711 fd.close() 712 action.data = lambda: file(fname, "rb") 713 action.attrs["pkg.size"] = str(os.stat(fname).st_size) 714 if action.name == "license": 715 action.attrs["transaction_id"] = transaction_id 716 # 717 # fmris may not be completely specified; enhance them to current 718 # version if this is the case 719 # 720 for attr in action.attrs: 721 if attr == "fmri" and \ 722 "@" not in action.attrs[attr] and \ 723 action.attrs[attr][5:] in pkgdict: 724 action.attrs[attr] += "@%s" % \ 725 pkgdict[action.attrs[attr][5:]].version 726 t.add(action) 727 728 if pkg.desc: 729 print " %s add set description=%s" % (pkg.name, pkg.desc) 730 action = actions.attribute.AttributeAction(None, 731 description = pkg.desc) 732 t.add(action) 733 734 if pkg.classification: 735 print " %s add set info.classification=%s" % \ 736 (pkg.name, pkg.classification) 737 attrs = dict(name="info.classification", 738 value=pkg.classification) 739 action = actions.attribute.AttributeAction(None, **attrs) 740 t.add(action) 741 742 if pkg.name != "SUNWipkg": 743 for p in pkg.srcpkgs: 744 try: 745 sp = svr4pkgsseen[p] 746 except KeyError: 747 continue 748 749 wanted_attrs = ( 750 "PKG", "NAME", "ARCH", "VERSION", "CATEGORY", 751 "VENDOR", "DESC", "HOTLINE" 752 ) 753 attrs = dict( 754 (k.lower(), v) 755 for k, v in sp.pkginfo.iteritems() 756 if k in wanted_attrs 757 ) 758 attrs["pkg"] = sp.pkginfo["PKG.PLAT"] 759 760 action = actions.legacy.LegacyAction(None, **attrs) 761 762 print " %s add %s" % (pkg.name, action) 763 t.add(action) 764 765 if undeps: 766 print "Missing dependencies:", list(undeps) 767 768 print " close" 769 pkg_fmri, pkg_state = t.close(refresh_index=not defer_refresh) 770 print "%s: %s\n" % (pkg_fmri, pkg_state) 771 772 def process_link_dependencies(path, target): 773 orig_target = target 774 if target[0] != "/": 775 target = os.path.normpath( 776 os.path.join(os.path.split(path)[0], target)) 777 778 if target in usedlist: 779 if show_debug: 780 print "hardlink %s -> %s makes %s depend on %s" % \ 781 ( 782 path, orig_target, 783 usedlist[path][1].name, 784 usedlist[target][1].name 785 ) 786 return ["%s@%s" % (usedlist[target][1].name, 787 usedlist[target][1].version)] 788 else: 789 return [] 790 791 def process_dependencies(fname, path): 792 if not elf.is_elf_object(fname): 793 return process_non_elf_dependencies(fname, path) 794 795 ei = elf.get_info(fname) 796 try: 797 ed = elf.get_dynamic(fname) 798 except elf.ElfError: 799 deps = [] 800 rp = [] 801 else: 802 deps = [ 803 d[0] 804 for d in ed.get("deps", []) 805 ] 806 rp = ed.get("runpath", "").split(":") 807 if len(rp) == 1 and rp[0] == "": 808 rp = [] 809 810 rp = [ 811 os.path.normpath(p.replace("$ORIGIN", "/" + os.path.dirname(path))) 812 for p in rp 813 ] 814 815 kernel64 = None 816 817 # For kernel modules, default path resolution is /platform/<platform>, 818 # /kernel, /usr/kernel. But how do we know what <platform> would be for 819 # a given module? Does it do fallbacks to, say, sun4u? 820 if path.startswith("kernel") or path.startswith("usr/kernel") or \ 821 (path.startswith("platform") and path.split("/")[2] == "kernel"): 822 if rp: 823 print "RUNPATH set for kernel module (%s): %s" % \ 824 (path, rp) 825 # Default kernel search path 826 rp.extend(("/kernel", "/usr/kernel")) 827 # What subdirectory should we look in for 64-bit kernel modules? 828 if ei["bits"] == 64: 829 if ei["arch"] == "i386": 830 kernel64 = "amd64" 831 elif ei["arch"] == "sparc": 832 kernel64 = "sparcv9" 833 else: 834 print ei["arch"] 835 else: 836 if "/lib" not in rp: 837 rp.append("/lib") 838 if "/usr/lib" not in rp: 839 rp.append("/usr/lib") 840 841 # XXX Do we need to handle anything other than $ORIGIN? x86 images have 842 # a couple of $PLATFORM and $ISALIST instances. 843 for p in rp: 844 if "$" in p: 845 tok = p[p.find("$"):] 846 if "/" in tok: 847 tok = tok[:tok.find("/")] 848 print "%s has dynamic token %s in rpath" % (path, tok) 849 850 dep_pkgs = [] 851 undeps = [] 852 depend_list = [] 853 for d in deps: 854 for p in rp: 855 # The instances of "[1:]" below are because usedlist 856 # stores paths without leading slash 857 if kernel64: 858 # Find 64-bit modules the way krtld does. 859 # XXX We don't resolve dependencies found in 860 # /platform, since we don't know where under 861 # /platform to look. 862 head, tail = os.path.split(d) 863 deppath = os.path.join(p, 864 head, 865 kernel64, 866 tail)[1:] 867 else: 868 # This is a hack for when a runpath uses the 64 869 # symlink to the actual 64-bit directory. 870 # Better would be to see if the runpath was a 871 # link, and if so, use its resolution, but 872 # extracting that information from used list is 873 # a pain, especially because you potentially 874 # have to resolve symlinks at all levels of the 875 # path. 876 if p.endswith("/64"): 877 if ei["arch"] == "i386": 878 p = p[:-2] + "amd64" 879 elif ei["arch"] == "sparc": 880 p = p[:-2] + "sparcv9" 881 deppath = os.path.join(p, d)[1:] 882 if deppath in usedlist: 883 dep_pkgs += [ "%s@%s" % 884 (usedlist[deppath][1].name, 885 usedlist[deppath][1].version) ] 886 depend_list.append( 887 ( 888 deppath, 889 usedlist[deppath][1].name 890 ) 891 ) 892 break 893 else: 894 undeps += [ d ] 895 896 if show_debug: 897 print "%s makes %s depend on %s" % \ 898 (path, usedlist[path][1].name, depend_list) 899 900 return dep_pkgs, undeps 901 902 def process_non_elf_dependencies(localpath, path): 903 # localpath is path to actual file 904 # path is path in installed image 905 # take 1 906 dep_pkgs = [] 907 undeps = [] 908 909 f = file(localpath) 910 l = f.readline() 911 f.close() 912 913 # add #!/ dependency 914 if l.startswith("#!/"): 915 # usedlist omits leading / 916 p = (l[2:].split()[0]) # first part of string is path (removes options) 917 # we don't handle dependencies through links, so fix up the common one 918 if p.startswith("/bin"): 919 p = "/usr" + p 920 if p[1:] in usedlist: 921 dep_pkgs += [ "%s@%s" % ( 922 usedlist[p[1:]][1].name, 923 usedlist[p[1:]][1].version) 924 ] 925 print "Added dependency on %s because of %s" % (usedlist[p[1:]][1].name, p) 926 else: 927 undeps = [ p ] 928 929 return dep_pkgs, undeps 930 931 def zap_strings(instr, strings): 932 """takes an input string and a list of strings to be removed, ignoring 933 case""" 934 for s in strings: 935 ls = s.lower() 936 while True: 937 li = instr.lower() 938 i = li.find(ls) 939 if i < 0: 940 break 941 instr = instr[0:i] + instr[i + len(ls):] 942 return instr 943 944 def get_branch(name): 945 return branch_dict.get(name, def_branch) 946 947 def_vers = "0.5.11" 948 def_branch = "" 949 def_wos_path = ["/net/netinstall.eng/export/nv/x/latest/Solaris_11/Product"] 950 create_repo = False 951 nopublish = False 952 show_debug = False 953 print_pkg_names = False 954 def_repo = "http://localhost:10000" 955 wos_path = [] 956 include_path = [] 957 branch_dict = {} 958 timestamp_files = [] 959 960 # 961 # files (by path) we always delete for bulk imports 962 # note that we ignore these if specifically included. 963 # 964 elided_files = {} 965 # 966 # if user uses -j, just_these_pkgs becomes list of pkgs to process 967 # allowing other arguments to be read in as files... 968 # 969 just_these_pkgs = [] 970 # 971 # strings to rip out of descriptions (case insensitve) 972 # 973 description_detritus = [", (usr)", ", (root)", " (usr)", " (root)", 974 " (/usr)", " - / filesystem", ",root(/)"] 975 # 976 # list of global includes to add to every package 977 # 978 global_includes = [] 979 # list of macro substitutions 980 macro_definitions = {} 981 982 try: 983 _opts, _args = getopt.getopt(sys.argv[1:], "B:D:I:G:NT:b:dj:m:ns:v:w:p:") 984 except getopt.GetoptError, _e: 985 print "unknown option", _e.opt 986 sys.exit(1) 987 988 g_proto_area = os.environ.get("ROOT", "") 989 990 for opt, arg in _opts: 991 if opt == "-b": 992 def_branch = arg.rstrip("abcdefghijklmnopqrstuvwxyz") 993 elif opt == "-d": 994 show_debug = True 995 elif opt == "-j": # means we're using the new argument form... 996 just_these_pkgs.append(arg) 997 elif opt == "-m": 998 _a = arg.split("=", 1) 999 macro_definitions.update([("$(%s)" % _a[0], _a[1])]) 1000 elif opt == "-n": 1001 nopublish = True 1002 elif opt == "-p": 1003 if not os.path.exists(arg): 1004 raise RuntimeError("Invalid prototype area specified.") 1005 # Clean up relative ../../, etc. out of path to proto 1006 g_proto_area = os.path.realpath(arg) 1007 elif opt == "-s": 1008 def_repo = arg 1009 if def_repo.startswith("file://"): 1010 # When publishing to file:// repositories, automatically 1011 # create the target repository if needed. 1012 create_repo = True 1013 elif opt == "-v": 1014 def_vers = arg 1015 elif opt == "-w": 1016 wos_path.append(arg) 1017 elif opt == "-D": 1018 elided_files[arg] = True 1019 elif opt == "-I": 1020 include_path.extend(arg.split(":")) 1021 elif opt == "-B": 1022 branch_file = file(arg) 1023 for _line in branch_file: 1024 if not _line.startswith("#"): 1025 bfargs = _line.split() 1026 if len(bfargs) == 2: 1027 branch_dict[bfargs[0]] = bfargs[1] 1028 branch_file.close() 1029 elif opt == "-G": #another file of global includes 1030 global_includes.append(arg) 1031 elif opt == "-N": 1032 print_pkg_names = True 1033 elif opt == "-T": 1034 timestamp_files.append(arg) 1035 1036 if not def_branch: 1037 print "need a branch id (build number)" 1038 sys.exit(1) 1039 elif "." not in def_branch: 1040 print "branch id needs to be of the form 'x.y'" 1041 sys.exit(1) 1042 1043 if not _args: 1044 print "need argument!" 1045 sys.exit(1) 1046 1047 if not wos_path: 1048 wos_path = def_wos_path 1049 1050 if just_these_pkgs: 1051 filelist = _args 1052 else: 1053 filelist = _args[0:1] 1054 just_these_pkgs = _args[1:] 1055 1056 1057 in_multiline_import = False 1058 1059 # This maps what files we've seen to a tuple of what packages they came from and 1060 # what packages they went into, so we can prevent more than one package from 1061 # grabbing the same file. 1062 usedlist = {} 1063 1064 # 1065 # pkgdict contains ipkgs by name 1066 # 1067 pkgdict = {} 1068 1069 # 1070 # destpkgs contains the list of ipkgs generated from each svr4 pkg 1071 # this is needed to generate metaclusters 1072 # 1073 destpkgs = {} 1074 1075 # 1076 #svr4 pkgs seen - pkgs indexed by name 1077 # 1078 svr4pkgsseen = {} 1079 1080 # 1081 #paths where we found the packages we need 1082 # 1083 svr4pkgpaths = {} 1084 1085 # 1086 # editable files and where they're found 1087 # 1088 editable_files = {} 1089 1090 # 1091 # hollow svr4 packages processed 1092 # 1093 hollow_pkgs = {} 1094 1095 1096 reuse_err = \ 1097 "Conflict in path %s: IPS %s SVR4 %s from %s with IPS %s SVR4 %s from %s" 1098 1099 1100 # First pass: don't actually publish anything, because we're not collecting 1101 # dependencies here. 1102 def read_full_line(lexer, continuation='\\'): 1103 """Read a complete line, allowing for the possibility of it being 1104 continued over multiple lines. Returns a single joined line, with 1105 continuation characters and leading and trailing spaces removed. 1106 """ 1107 1108 lines = [] 1109 while True: 1110 line = lexer.instream.readline().strip() 1111 lexer.lineno = lexer.lineno + 1 1112 if line[-1] in continuation: 1113 lines.append(line[:-1]) 1114 else: 1115 lines.append(line) 1116 break 1117 1118 return apply_macros(' '.join(lines)) 1119 1120 def apply_macros(s): 1121 """Apply macro subs defined on command line... keep applying 1122 macros until no translations are found. If macro translates 1123 to a comment, replace entire token text.""" 1124 while s and "$(" in s: 1125 for key in macro_definitions.keys(): 1126 if key in s: 1127 value = macro_definitions[key] 1128 if value == "#": # comment character 1129 s = "#" # affects whole token 1130 break 1131 s = s.replace(key, value) 1132 break # look for more substitutions 1133 else: 1134 break # no more substitutable tokens 1135 return s 1136 1137 def sourcehook(filename): 1138 """ implement include hierarchy """ 1139 for i in include_path: 1140 f = os.path.join(i, filename) 1141 if os.path.exists(f): 1142 return (f, open(f)) 1143 1144 return filename, open(filename) 1145 1146 class tokenlexer(shlex.shlex): 1147 def read_token(self): 1148 """ simple replacement of $(ARCH) with a non-special 1149 value defined on the command line is trivial. Since 1150 shlex's read_token routine also strips comments and 1151 white space, this read_token cannot return either 1152 one so any macros that translate to either spaces or 1153 # (comment) need to be removed from the token stream.""" 1154 1155 while True: 1156 s = apply_macros(shlex.shlex.read_token(self)) 1157 if s == "#": # discard line if comment; try again 1158 self.instream.readline() 1159 self.lineno = self.lineno + 1 1160 # bail on EOF or not space; loop on space 1161 elif s == None or (s != "" and not s.isspace()): 1162 break 1163 return s 1164 1165 curpkg = None 1166 def SolarisParse(mf): 1167 global curpkg 1168 global in_multiline_import 1169 1170 lexer = tokenlexer(file(mf), mf, True) 1171 lexer.whitespace_split = True 1172 lexer.source = "include" 1173 lexer.sourcehook = sourcehook 1174 1175 while True: 1176 token = lexer.get_token() 1177 1178 if not token: 1179 break 1180 1181 if token == "package": 1182 curpkg = start_package(lexer.get_token()) 1183 1184 if print_pkg_names: 1185 print "-j %s" % curpkg.name 1186 1187 elif token == "end": 1188 endarg = lexer.get_token() 1189 if endarg == "package": 1190 if print_pkg_names: 1191 curpkg = None 1192 continue 1193 1194 for filename in global_includes: 1195 for i in include_path: 1196 f = os.path.join(i, filename) 1197 if os.path.exists(f): 1198 SolarisParse(f) 1199 break 1200 else: 1201 raise RuntimeError("File not " 1202 "found: %s" % filename) 1203 try: 1204 end_package(curpkg) 1205 except Exception, e: 1206 print "ERROR(end_pkg):", e 1207 1208 curpkg = None 1209 if endarg == "import": 1210 in_multiline_import = False 1211 curpkg.imppkg = None 1212 1213 elif token == "version": 1214 curpkg.version = lexer.get_token() 1215 1216 elif token == "import": 1217 package_name = lexer.get_token() 1218 next = lexer.get_token() 1219 if next != "exclude": 1220 line = "" 1221 lexer.push_token(next) 1222 else: 1223 line = read_full_line(lexer) 1224 1225 if not print_pkg_names: 1226 curpkg.import_pkg(package_name, line) 1227 1228 elif token == "from": 1229 pkgspec = lexer.get_token() 1230 if not print_pkg_names: 1231 p = SolarisPackage(pkg_path(pkgspec)) 1232 curpkg.imppkg = p 1233 spkgname = p.pkginfo["PKG.PLAT"] 1234 svr4pkgpaths[spkgname] = pkg_path(pkgspec) 1235 svr4pkgsseen[spkgname] = p 1236 curpkg.add_svr4_src(spkgname) 1237 1238 junk = lexer.get_token() 1239 assert junk == "import" 1240 in_multiline_import = True 1241 1242 elif token == "classification": 1243 cat_subcat = lexer.get_token() 1244 curpkg.classification = \ 1245 "org.opensolaris.category.2008:%s" % cat_subcat 1246 1247 elif token == "description": 1248 curpkg.desc = lexer.get_token() 1249 1250 elif token == "depend": 1251 curpkg.depend.append(lexer.get_token()) 1252 1253 elif token == "depend_path": 1254 curpkg.file_depend.append(lexer.get_token()) 1255 1256 elif token == "cluster": 1257 curpkg.add_svr4_src(lexer.get_token()) 1258 1259 elif token == "idepend": 1260 curpkg.idepend.append(lexer.get_token()) 1261 1262 elif token == "undepend": 1263 curpkg.undepend.append(lexer.get_token()) 1264 1265 elif token == "add": 1266 curpkg.extra.append(read_full_line(lexer)) 1267 1268 elif token == "drop": 1269 f = lexer.get_token() 1270 if print_pkg_names: 1271 continue 1272 l = [o for o in curpkg.files if o.pathname == f] 1273 if not l: 1274 print "Cannot drop '%s' from '%s': not " \ 1275 "found" % (f, curpkg.name) 1276 else: 1277 del curpkg.files[curpkg.files.index(l[0])] 1278 # XXX The problem here is that if we do this on 1279 # a shared file (directory, etc), then it's 1280 # missing from usedlist entirely, since we don't 1281 # keep around *all* packages delivering a shared 1282 # file, just the last seen. This probably 1283 # doesn't matter much. 1284 del usedlist[f] 1285 1286 elif token == "drop_license": 1287 curpkg.dropped_licenses.append(lexer.get_token()) 1288 1289 elif token == "chattr": 1290 fname = lexer.get_token() 1291 line = read_full_line(lexer) 1292 if print_pkg_names: 1293 continue 1294 try: 1295 curpkg.chattr(fname, line) 1296 except Exception, e: 1297 print "Can't change attributes on " + \ 1298 "'%s': not in the package" % fname, e 1299 raise 1300 1301 elif token == "chattr_glob": 1302 glob = lexer.get_token() 1303 line = read_full_line(lexer) 1304 if print_pkg_names: 1305 continue 1306 try: 1307 curpkg.chattr_glob(glob, line) 1308 except Exception, e: 1309 print "Can't change attributes on " + \ 1310 "'%s': no matches in the package" % \ 1311 glob, e 1312 raise 1313 1314 elif in_multiline_import: 1315 next = lexer.get_token() 1316 if next == "with": 1317 # I can't imagine this is supported, but there's 1318 # no other way to read the rest of the line 1319 # without a whole lot more pain. 1320 line = read_full_line(lexer) 1321 else: 1322 lexer.push_token(next) 1323 line = "" 1324 1325 try: 1326 curpkg.import_file(token, line) 1327 except Exception, e: 1328 print "ERROR(import_file):", e 1329 raise 1330 else: 1331 raise RuntimeError("Error: unknown token '%s' " 1332 "(%s:%s)" % (token, lexer.infile, lexer.lineno)) 1333 1334 if print_pkg_names: 1335 for _mf in filelist: 1336 SolarisParse(_mf) 1337 sys.exit(0) 1338 1339 1340 print "First pass:", datetime.now() 1341 1342 for _mf in filelist: 1343 SolarisParse(_mf) 1344 1345 seenpkgs = set(i[0] for i in usedlist.values()) 1346 1347 print "Files you seem to have forgotten:\n " + "\n ".join( 1348 "%s %s" % (f.type, f.pathname) 1349 for pkg in seenpkgs 1350 for f in svr4pkgsseen[pkg].manifest 1351 if f.type != "i" and f.pathname not in usedlist) 1352 1353 print "\n\nDuplicate Editables files list:\n" 1354 1355 if editable_files: 1356 length = 2 + max(len(p) for p in editable_files) 1357 for paths in editable_files: 1358 if len(editable_files[paths]) > 1: 1359 print ("%s:" % paths).ljust(length - 1) + \ 1360 ("\n".ljust(length)).join("%s (from %s)" % \ 1361 (l[1].name, l[0]) for l in editable_files[paths]) 1362 1363 1364 # Second pass: iterate over the existing package objects, gathering dependencies 1365 # and publish! 1366 1367 print "Second pass:", datetime.now() 1368 1369 print "New packages:\n" 1370 # XXX Sort these. Preferably topologically, if possible, alphabetically 1371 # otherwise (for a rough progress gauge). 1372 if just_these_pkgs: 1373 newpkgs = set(pkgdict[name] 1374 for name in pkgdict.keys() 1375 if name in just_these_pkgs 1376 ) 1377 else: 1378 newpkgs = set(pkgdict.values()) 1379 1380 # Indicates whether search indices refresh will be deferred until the end. 1381 defer_refresh = False 1382 # Indicates whether local publishing is active. 1383 local_publish = False 1384 if def_repo.startswith("file:"): 1385 # If publishing to disk, the search indices should be refreshed at 1386 # the end of the publishing process and the feed cache will have to be 1387 # generated by starting the depot server using the provided path and 1388 # then accessing it. 1389 defer_refresh = True 1390 local_publish = True 1391 1392 processed = 0 1393 total = len(newpkgs) 1394 for _p in sorted(newpkgs): 1395 print "Package '%s'" % _p.name 1396 print " Version:", _p.version 1397 print " Description:", _p.desc 1398 print " Classification:", _p.classification 1399 try: 1400 publish_pkg(_p) 1401 except trans.TransactionError, _e: 1402 print "%s: FAILED: %s\n" % (_p.name, _e) 1403 processed += 1 1404 print "%d/%d packages processed; %.2f%% complete" % (processed, total, 1405 processed * 100.0 / total) 1406 1407 if not nopublish and defer_refresh: 1408 # This has to be done at the end for some publishing modes. 1409 print "Updating search indices..." 1410 _t = trans.Transaction(def_repo) 1411 _t.refresh_index() 1412 1413 # Ensure that the feed is updated and cached to reflect changes. 1414 if not nopublish: 1415 print "Caching RSS/Atom feed..." 1416 dc = None 1417 durl = def_repo 1418 if local_publish: 1419 # The depot server isn't already running, so will have to be 1420 # temporarily started to allow proper feed cache generation. 1421 dc = depotcontroller.DepotController() 1422 dc.set_depotd_path(g_proto_area + "/usr/lib/pkg.depotd") 1423 dc.set_depotd_content_root(g_proto_area + "/usr/share/lib/pkg") 1424 1425 _scheme, _netloc, _path, _params, _query, _fragment = \ 1426 urlparse.urlparse(def_repo, "file", allow_fragments=0) 1427 1428 dc.set_repodir(_path) 1429 1430 # XXX There must be a better way... 1431 dc.set_port(29083) 1432 1433 # Start the depot 1434 dc.start() 1435 1436 durl = "http://localhost:29083" 1437 1438 _f = urllib.urlopen("%s/feed" % durl) 1439 _f.close() 1440 1441 if dc: 1442 dc.stop() 1443 dc = None 1444 1445 print "Done:", datetime.now() 1446
Indexes created Sun Nov 22 02:16:54 UTC 2009
Terms of Use |
Privacy |
Trademarks |
Copyright Policy |
Site Guidelines |
Help
Your use of this web site or any of its content or software indicates your agreement to be bound by these Terms of Use.
Copyright © 1995-2008 Sun Microsystems, Inc.