On Wed, Jul 25, 2012 at 10:09:22PM +0100, Richard Purdie wrote: > Currently all sstate files are placed into one directory. This does not scale > and > causes a variety of filesystem issues. This patch adds a two character > subdirectory > to the layout (based on the first two characters of the hash) so that files > can be split into several directories. > > This should help performance of sstate in most cases by avoding creating > directories with > huge numbers of files. > > The SSTATE_MIRRORS syntax needs updating to account for the extra path > element by > the addition of a PATH item, for example: > > SSTATE_MIRRORS = "file://.* file:///some/path/to/sstate-cache/PATH" > SSTATE_MIRRORS = "file://.* http://192.168.1.23/sstate-cache/PATH"; > > This change also sets the scene for using things like lsb-release in > the
Is it possible to create 2nd level cache with this? I have some server with slow upload but fully populated sstate-cache. So on server with faster upload which could be used as offical SSTATE_MIRROR for SHR distro I would like to add SSTATE_MIRRORS ?= "file://.* http://slow-server/sstate-cache/PATH"; And then sync my sstate-cache directory to public accessible web root (with rsync). Problem is that now sstate-cache has all files in slightly different layout then original sstate-cache on slow server. From what I see I guess it finds URL with correct prefix "sstate-cache/Gentoo-2.1/0d" and downloads it directly to sstate-cache dir (and adds .done) OE @ ~/oe-core $ ll sstate-cache/sstate-apr-native-x86_64-linux-1.4.6-r1-x86_64-2-*populate-lic* -rw-r--r-- 1 bitbake bitbake 9257 Jul 30 12:31 sstate-cache/sstate-apr-native-x86_64-linux-1.4.6-r1-x86_64-2-0d2ed24b90d50bf83e5fe94536596e50_populate-lic.tgz -rw-r--r-- 1 bitbake bitbake 0 Aug 2 15:40 sstate-cache/sstate-apr-native-x86_64-linux-1.4.6-r1-x86_64-2-0d2ed24b90d50bf83e5fe94536596e50_populate-lic.tgz.done And then creates symlink in right prefix back to absolute path of sstate-cache/file: OE @ ~/oe-core $ ll sstate-cache/Gentoo-2.1/0d/sstate-apr-native-x86_64-linux-1.4.6-r1-x86_64-2-*populate-lic* lrwxrwxrwx 1 bitbake bitbake 123 Aug 2 15:40 sstate-cache/Gentoo-2.1/0d/sstate-apr-native-x86_64-linux-1.4.6-r1-x86_64-2-0d2ed24b90d50bf83e5fe94536596e50_populate-lic.tgz -> /OE/oe-core/sstate-cache/sstate-apr-native-x86_64-linux-1.4.6-r1-x86_64-2-0d2ed24b90d50bf83e5fe94536596e50_populate-lic.tgz But after sstate-cache directory is rsynced somewhere else and oe-core/sstate-cache is removed, all those symlinks point nowhere and public sstate-cache is unusable. Can we have relative paths used in symlinks or even instruct fetcher to download that file directly to right prefix? Cheers, > Signed-off-by: Richard Purdie <richard.pur...@linuxfoundation.org> > --- > diff --git a/meta/classes/sstate.bbclass b/meta/classes/sstate.bbclass > index 570b371..d00779a 100644 > --- a/meta/classes/sstate.bbclass > +++ b/meta/classes/sstate.bbclass > @@ -4,16 +4,21 @@ SSTATE_MANIFESTS ?= "${TMPDIR}/sstate-control" > SSTATE_MANFILEBASE = "${SSTATE_MANIFESTS}/manifest-${SSTATE_MANMACH}-" > SSTATE_MANFILEPREFIX = "${SSTATE_MANFILEBASE}${PN}" > > +def generate_sstatefn(spec, hash, d): > + if not hash: > + hash = "INVALID" > + return hash[:2] + "/" + spec + hash > > SSTATE_PKGARCH = "${PACKAGE_ARCH}" > SSTATE_PKGSPEC = > "sstate-${PN}-${PACKAGE_ARCH}${TARGET_VENDOR}-${TARGET_OS}-${PV}-${PR}-${SSTATE_PKGARCH}-${SSTATE_VERSION}-" > -SSTATE_PKGNAME = "${SSTATE_PKGSPEC}${BB_TASKHASH}" > +SSTATE_PKGNAME = "${@generate_sstatefn(d.getVar('SSTATE_PKGSPEC', True), > d.getVar('BB_TASKHASH', True), d)}" > SSTATE_PKG = "${SSTATE_DIR}/${SSTATE_PKGNAME}" > +SSTATE_PATHSPEC = "${SSTATE_DIR}/*/${SSTATE_PKGSPEC}" > > SSTATE_SCAN_FILES ?= "*.la *-config *_config" > SSTATE_SCAN_CMD ?= 'find ${SSTATE_BUILDDIR} \( -name "${@"\" -o -name > \"".join(d.getVar("SSTATE_SCAN_FILES", True).split())}" \) -type f' > > -BB_HASHFILENAME = "${SSTATE_PKGNAME}" > +BB_HASHFILENAME = "${SSTATE_PKGSPEC}" > > SSTATE_MANMACH ?= "${SSTATE_PKGARCH}" > > @@ -158,10 +163,11 @@ def sstate_installpkg(ss, d): > oe.path.remove(dir) > > sstateinst = d.expand("${WORKDIR}/sstate-install-%s/" % ss['name']) > + sstatefetch = d.getVar('SSTATE_PKGNAME', True) + '_' + ss['name'] + > ".tgz" > sstatepkg = d.getVar('SSTATE_PKG', True) + '_' + ss['name'] + ".tgz" > > if not os.path.exists(sstatepkg): > - pstaging_fetch(sstatepkg, d) > + pstaging_fetch(sstatefetch, sstatepkg, d) > > if not os.path.isfile(sstatepkg): > bb.note("Staging package %s does not exist" % sstatepkg) > @@ -223,8 +229,7 @@ def sstate_installpkg(ss, d): > def sstate_clean_cachefile(ss, d): > import oe.path > > - sstatepkgdir = d.getVar('SSTATE_DIR', True) > - sstatepkgfile = sstatepkgdir + '/' + d.getVar('SSTATE_PKGSPEC', True) + > "*_" + ss['name'] + ".tgz*" > + sstatepkgfile = d.getVar('SSTATE_PATHSPEC', True) + "*_" + ss['name'] + > ".tgz*" > bb.note("Removing %s" % sstatepkgfile) > oe.path.remove(sstatepkgfile) > > @@ -417,7 +422,7 @@ def sstate_package(ss, d): > > return > > -def pstaging_fetch(sstatepkg, d): > +def pstaging_fetch(sstatefetch, sstatepkg, d): > import bb.fetch2 > > # Only try and fetch if the user has configured a mirror > @@ -430,7 +435,7 @@ def pstaging_fetch(sstatepkg, d): > bb.data.update_data(localdata) > > dldir = localdata.expand("${SSTATE_DIR}") > - srcuri = "file://" + os.path.basename(sstatepkg) > + srcuri = "file://" + sstatefetch > > bb.mkdirhier(dldir) > > @@ -519,8 +524,7 @@ def sstate_checkhashes(sq_fn, sq_task, sq_hash, > sq_hashfn, d): > } > > for task in range(len(sq_fn)): > - sstatefile = d.expand("${SSTATE_DIR}/" + sq_hashfn[task] + "_" + > mapping[sq_task[task]] + ".tgz") > - sstatefile = sstatefile.replace("${BB_TASKHASH}", sq_hash[task]) > + sstatefile = d.expand("${SSTATE_DIR}/" + > generate_sstatefn(sq_hashfn[task], sq_hash[task], d) + "_" + > mapping[sq_task[task]] + ".tgz") > if os.path.exists(sstatefile): > bb.debug(2, "SState: Found valid sstate file %s" % sstatefile) > ret.append(task) > @@ -544,10 +548,9 @@ def sstate_checkhashes(sq_fn, sq_task, sq_hash, > sq_hashfn, d): > if task in ret: > continue > > - sstatefile = d.expand("${SSTATE_DIR}/" + sq_hashfn[task] + "_" + > mapping[sq_task[task]] + ".tgz") > - sstatefile = sstatefile.replace("${BB_TASKHASH}", sq_hash[task]) > + sstatefile = d.expand(generate_sstatefn(sq_hashfn[task], > sq_hash[task], d) + "_" + mapping[sq_task[task]] + ".tgz") > > - srcuri = "file://" + os.path.basename(sstatefile) > + srcuri = "file://" + sstatefile > localdata.setVar('SRC_URI', srcuri) > bb.debug(2, "SState: Attempting to fetch %s" % srcuri) > > > > > > _______________________________________________ > Openembedded-devel mailing list > openembedded-de...@lists.openembedded.org > http://lists.linuxtogo.org/cgi-bin/mailman/listinfo/openembedded-devel -- Martin 'JaMa' Jansa jabber: martin.ja...@gmail.com
signature.asc
Description: Digital signature
_______________________________________________ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.linuxtogo.org/cgi-bin/mailman/listinfo/openembedded-core
