From 07b2af1a45c5276125b54d64f839fdcd660d602d Mon Sep 17 00:00:00 2001 From: Maxime Petazzoni Date: Thu, 2 Sep 2010 12:09:44 +0200 Subject: Remove the spider feature In preparation for the re-work of the DOWNLOAD helper to support multiple download methods and protocols, the spider feature used with wget is removed for now until it is re-implemented on top of the new download methods. Signed-off-by: Maxime Petazzoni Signed-off-by: Thomas Petazzoni --- package/Makefile.package.in | 10 ---------- 1 file changed, 10 deletions(-) (limited to 'package/Makefile.package.in') diff --git a/package/Makefile.package.in b/package/Makefile.package.in index d76b7913d..ba0fd8433 100644 --- a/package/Makefile.package.in +++ b/package/Makefile.package.in @@ -81,16 +81,11 @@ TERM_RESET := $(shell tput rmso) # $(call DOWNLOAD,$(FOO_SITE),$(FOO_SOURCE)) ################################################################################ -# support make source-check/external-deps -ifneq ($(SPIDER),) -DOWNLOAD=$(WGET) -P $(DL_DIR) $(1)/$(2) -else define DOWNLOAD $(Q)test -e $(DL_DIR)/$(2) || \ for site in $(call qstrip,$(BR2_PRIMARY_SITE)) $(1) $(call qstrip,$(BR2_BACKUP_SITE)); \ do $(WGET) -P $(DL_DIR) $$site/$(2) && exit; done endef -endif # Utility programs used to build packages TAR ?= tar @@ -117,19 +112,14 @@ endef # Retrieve the archive $(BUILD_DIR)/%/.stamp_downloaded: -# support make source-check/external-deps -ifeq ($(SPIDER),) # Only show the download message if it isn't already downloaded $(Q)(test -e $(DL_DIR)/$($(PKG)_SOURCE) && \ (test -z $($(PKG)_PATCH) || test -e $(DL_DIR)$($(PKG)_PATCH))) || \ $(call MESSAGE,"Downloading") -endif $(call DOWNLOAD,$($(PKG)_SITE),$($(PKG)_SOURCE)) $(if $($(PKG)_PATCH),$(call DOWNLOAD,$($(PKG)_SITE),$($(PKG)_PATCH))) -ifeq ($(SPIDER),) $(Q)mkdir -p $(@D) $(Q)touch $@ -endif # Unpack the archive $(BUILD_DIR)/%/.stamp_extracted: -- cgit v1.2.3 From 993e51bc22f508fbaf39c5c49fd0595fc5c56013 Mon Sep 17 00:00:00 2001 From: Maxime Petazzoni Date: Thu, 2 Sep 2010 12:09:47 +0200 Subject: Implement basic non-wget download methods Packages can now be sourced from Git and Subversion repositories. The download method will be autodetected from the URI (git://, svn://, etc). If the repository is accessed through http(s), you can force the download method by setting a _SITE_METHOD variable to either 'git' or 'svn', respectively and without the quotes. The package's _VERSION variable defines which commit, revision, tag or branch should be checked out. For Git, it can be HEAD, a commit ID, a tag name or branch name (anything that can be checked out with `git checkout`). For Subversion, it must be a revision number, or HEAD. Signed-off-by: Maxime Petazzoni Signed-off-by: Thomas Petazzoni --- package/Makefile.package.in | 68 ++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 64 insertions(+), 4 deletions(-) (limited to 'package/Makefile.package.in') diff --git a/package/Makefile.package.in b/package/Makefile.package.in index ba0fd8433..5444472ad 100644 --- a/package/Makefile.package.in +++ b/package/Makefile.package.in @@ -68,6 +68,35 @@ MESSAGE = echo "$(TERM_BOLD)>>> $($(PKG)_NAME) $($(PKG)_VERSION) $(1)$(TERM_RESE TERM_BOLD := $(shell tput smso) TERM_RESET := $(shell tput rmso) +################################################################################ +# The DOWNLOAD_{GIT,SVN} helpers are in charge of getting a working copy of +# the source repository for their corresponding SCM, checking out the requested +# version / commit / tag, and create an archive out of it. +################################################################################ + +define DOWNLOAD_GIT + pushd $(DL_DIR) > /dev/null && \ + $(GIT) clone $($(PKG)_SITE) $($(PKG)_BASE_NAME) && \ + pushd $($(PKG)_BASE_NAME) > /dev/null && \ + $(GIT) archive --format=tar --prefix=$($(PKG)_BASE_NAME)/ $($(PKG)_DL_VERSION) | \ + gzip -c > $(DL_DIR)/$($(PKG)_SOURCE) && \ + popd > /dev/null && \ + rm -rf $($(PKG)_DL_DIR) && \ + popd > /dev/null +endef + +define DOWNLOAD_SVN + pushd $(DL_DIR) > /dev/null && \ + $(SVN) export -r $($(PKG)_DL_VERSION) $($(PKG)_SITE) $($(PKG)_DL_DIR) && \ + $(TAR) czf $($(PKG)_SOURCE) $($(PKG)_BASE_NAME)/ && \ + rm -rf $($(PKG)_DL_DIR) && \ + popd > /dev/null +endef + +define DOWNLOAD_WGET + $(WGET) -P $(DL_DIR) $(call qstrip,$(1))/$(2) +endef + ################################################################################ # DOWNLOAD -- Download helper. Will try to download source from: # 1) BR2_PRIMARY_SITE if enabled @@ -83,8 +112,20 @@ TERM_RESET := $(shell tput rmso) define DOWNLOAD $(Q)test -e $(DL_DIR)/$(2) || \ - for site in $(call qstrip,$(BR2_PRIMARY_SITE)) $(1) $(call qstrip,$(BR2_BACKUP_SITE)); \ - do $(WGET) -P $(DL_DIR) $$site/$(2) && exit; done + (if test -n "$(call qstrip,$(BR2_PRIMARY_SITE))" ; then \ + $(call DOWNLOAD_WGET,$(BR2_PRIMARY_SITE),$(2)) && exit ; \ + fi ; \ + if test -n "$(1)" ; then \ + case "$($(PKG)_SITE_METHOD)" in \ + git) $(DOWNLOAD_GIT) && exit ;; \ + svn) $(DOWNLOAD_SVN) && exit ;; \ + *) $(call DOWNLOAD_WGET,$(1),$(2)) && exit ;; \ + esac ; \ + fi ; \ + if test -n "$(call qstrip,$(BR2_BACKUP_SITE))" ; then \ + $(call DOWNLOAD_WGET,$(BR2_BACKUP_SITE),$(2)) && exit ; \ + fi ; \ + exit 1) endef # Utility programs used to build packages @@ -244,13 +285,23 @@ ifndef $(2)_VERSION endif endif -$(2)_DIR = $$(BUILD_DIR)/$(1)-$$($(2)_VERSION) +# Keep the package version that may contain forward slashes in the _DL_VERSION +# variable, then replace all forward slashes ('/') by underscores ('_') to +# sanitize the package version that is used in paths, directory and file names. +# Forward slashes may appear in the package's version when pointing to a +# version control system branch or tag, for example remotes/origin/1_10_stable. +$(2)_DL_VERSION = $($(2)_VERSION) +$(2)_VERSION = $(subst /,_,$($(2)_VERSION)) + +$(2)_BASE_NAME = $(1)-$$($(2)_VERSION) +$(2)_DL_DIR = $$(DL_DIR)/$$($(2)_BASE_NAME) +$(2)_DIR = $$(BUILD_DIR)/$$($(2)_BASE_NAME) ifndef $(2)_SOURCE ifdef $(3)_SOURCE $(2)_SOURCE = $($(3)_SOURCE) else - $(2)_SOURCE ?= $(1)-$$($(2)_VERSION).tar.gz + $(2)_SOURCE ?= $$($(2)_BASE_NAME).tar.gz endif endif @@ -269,6 +320,15 @@ ifndef $(2)_SITE endif endif +ifndef $(2)_SITE_METHOD + ifdef $(3)_SITE_METHOD + $(2)_SITE_METHOD = $($(3)_SITE_METHOD) + else + # Try automatic detection using the scheme part of the URI + $(2)_SITE_METHOD = $(firstword $(subst ://, ,$(call qstrip,$($(2)_SITE)))) + endif +endif + $(2)_DEPENDENCIES ?= $(2)_INSTALL_STAGING ?= NO $(2)_INSTALL_TARGET ?= YES -- cgit v1.2.3 From d147b81fda4419ea7307835d39cdb58b965882aa Mon Sep 17 00:00:00 2001 From: Maxime Petazzoni Date: Thu, 2 Sep 2010 12:31:57 +0200 Subject: Re-implement the source-check and external-deps targets The new DL_MODE variable dispatches between the various download implementations of each method (Git, Subversion, Wget) to deal with the normal download (default mode, 'DOWNLOAD'), the source-check ('SOURCE_CHECK') and to show the external dependencies for external-deps ('SHOW_EXTERNAL_DEPS'). For the latter, the legacy script wget-show-external-deps.sh is no longer required as $(WGET) isn't called directly anymore but always through the DOWNLOAD helper. Signed-off-by: Maxime Petazzoni Signed-off-by: Thomas Petazzoni --- Makefile | 13 +----- package/Makefile.package.in | 86 ++++++++++++++++++++++++++++++------ package/mpfr/mpfr.mk | 2 + toolchain/wget-show-external-deps.sh | 6 --- 4 files changed, 77 insertions(+), 30 deletions(-) delete mode 100755 toolchain/wget-show-external-deps.sh (limited to 'package/Makefile.package.in') diff --git a/Makefile b/Makefile index 778040308..faa802bb7 100644 --- a/Makefile +++ b/Makefile @@ -244,19 +244,10 @@ ifeq ($(ARCH),xtensa) ARCH:=$(ARCH)_$(call qstrip,$(BR2_xtensa_core_name)) endif -WGET:=$(call qstrip,$(BR2_WGET)) $(QUIET) -SVN:=$(call qstrip,$(BR2_SVN)) $(QUIET) -BZR:=$(call qstrip,$(BR2_BZR)) $(QUIET) -GIT:=$(call qstrip,$(BR2_GIT)) $(QUIET) ZCAT:=$(call qstrip,$(BR2_ZCAT)) BZCAT:=$(call qstrip,$(BR2_BZCAT)) TAR_OPTIONS=$(call qstrip,$(BR2_TAR_OPTIONS)) -xf -DL_DIR=$(call qstrip,$(BR2_DL_DIR)) -ifeq ($(DL_DIR),) -DL_DIR:=$(TOPDIR)/dl -endif - GNU_TARGET_SUFFIX:=-$(call qstrip,$(BR2_GNU_TARGET_SUFFIX)) STAGING_DIR:=$(call qstrip,$(BR2_STAGING_DIR)) @@ -435,10 +426,10 @@ endif source: $(TARGETS_SOURCE) $(HOST_SOURCE) _source-check: - @echo "TODO $@" + $(MAKE) DL_MODE=SOURCE_CHECK $(EXTRAMAKEARGS) source external-deps: - @echo "TODO $@" + @$(MAKE) -Bs DL_MODE=SHOW_EXTERNAL_DEPS $(EXTRAMAKEARGS) source show-targets: @echo $(TARGETS) diff --git a/package/Makefile.package.in b/package/Makefile.package.in index 5444472ad..6f584a747 100644 --- a/package/Makefile.package.in +++ b/package/Makefile.package.in @@ -68,35 +68,92 @@ MESSAGE = echo "$(TERM_BOLD)>>> $($(PKG)_NAME) $($(PKG)_VERSION) $(1)$(TERM_RESE TERM_BOLD := $(shell tput smso) TERM_RESET := $(shell tput rmso) +# Download method commands +WGET:=$(call qstrip,$(BR2_WGET)) $(QUIET) +SVN:=$(call qstrip,$(BR2_SVN)) $(QUIET) +BZR:=$(call qstrip,$(BR2_BZR)) $(QUIET) +GIT:=$(call qstrip,$(BR2_GIT)) $(QUIET) + +# Default spider mode is 'DOWNLOAD'. Other possible values are 'SOURCE_CHECK' +# used by the _source-check target and 'SHOW_EXTERNAL_DEPS', used by the +# external-deps target. +DL_MODE=DOWNLOAD + +DL_DIR=$(call qstrip,$(BR2_DL_DIR)) +ifeq ($(DL_DIR),) +DL_DIR:=$(TOPDIR)/dl +endif + ################################################################################ # The DOWNLOAD_{GIT,SVN} helpers are in charge of getting a working copy of # the source repository for their corresponding SCM, checking out the requested -# version / commit / tag, and create an archive out of it. +# version / commit / tag, and create an archive out of it. DOWNLOAD_WGET is the +# normal wget-based download mechanism. +# +# The SOURCE_CHECK_{GIT,SVN,WGET} helpers are in charge of simply checking that +# the source is available for download. This can be used to make sure one will +# be able to get all the sources needed for one's build configuration. +# +# The SHOW_EXTERNAL_DEPS_{GIT,SVN,WGET} helpers simply output to the console +# the names of the files that will be downloaded, or path and revision of the +# source repositories, producing a list of all the "external dependencies" of +# a given build configuration. ################################################################################ define DOWNLOAD_GIT - pushd $(DL_DIR) > /dev/null && \ + test -e $(DL_DIR)/$($(PKG)_SOURCE) || \ + (pushd $(DL_DIR) > /dev/null && \ $(GIT) clone $($(PKG)_SITE) $($(PKG)_BASE_NAME) && \ pushd $($(PKG)_BASE_NAME) > /dev/null && \ $(GIT) archive --format=tar --prefix=$($(PKG)_BASE_NAME)/ $($(PKG)_DL_VERSION) | \ gzip -c > $(DL_DIR)/$($(PKG)_SOURCE) && \ popd > /dev/null && \ rm -rf $($(PKG)_DL_DIR) && \ - popd > /dev/null + popd > /dev/null) +endef + +# TODO: improve to check that the given PKG_DL_VERSION exists on the remote +# repository +define SOURCE_CHECK_GIT + $(GIT) ls-remote --heads $($(PKG)_SITE) > /dev/null +endef + +define SHOW_EXTERNAL_DEPS_GIT + echo "$($(PKG)_SITE) [git: $($(PKG)_DL_VERSION)]" endef + define DOWNLOAD_SVN - pushd $(DL_DIR) > /dev/null && \ + test -e $(DL_DIR)/$($(PKG)_SOURCE) || \ + (pushd $(DL_DIR) > /dev/null && \ $(SVN) export -r $($(PKG)_DL_VERSION) $($(PKG)_SITE) $($(PKG)_DL_DIR) && \ $(TAR) czf $($(PKG)_SOURCE) $($(PKG)_BASE_NAME)/ && \ rm -rf $($(PKG)_DL_DIR) && \ - popd > /dev/null + popd > /dev/null) +endef + +define SOURCE_CHECK_SVN + $(SVN) ls $($(PKG)_SITE) > /dev/null +endef + +define SHOW_EXTERNAL_DEPS_SVN + echo "$($(PKG)_SITE) [svn: $($(PKG)_DL_VERSION)]" endef + define DOWNLOAD_WGET + test -e $(DL_DIR)/$(2) || \ $(WGET) -P $(DL_DIR) $(call qstrip,$(1))/$(2) endef +define SOURCE_CHECK_WGET + $(WGET) --spider $(call qstrip,$(1))/$(2) +endef + +define SHOW_EXTERNAL_DEPS_WGET + echo $(2) +endef + ################################################################################ # DOWNLOAD -- Download helper. Will try to download source from: # 1) BR2_PRIMARY_SITE if enabled @@ -111,21 +168,20 @@ endef ################################################################################ define DOWNLOAD - $(Q)test -e $(DL_DIR)/$(2) || \ - (if test -n "$(call qstrip,$(BR2_PRIMARY_SITE))" ; then \ - $(call DOWNLOAD_WGET,$(BR2_PRIMARY_SITE),$(2)) && exit ; \ + $(Q)if test -n "$(call qstrip,$(BR2_PRIMARY_SITE))" ; then \ + $(call $(DL_MODE)_WGET,$(BR2_PRIMARY_SITE),$(2)) && exit ; \ fi ; \ if test -n "$(1)" ; then \ case "$($(PKG)_SITE_METHOD)" in \ - git) $(DOWNLOAD_GIT) && exit ;; \ - svn) $(DOWNLOAD_SVN) && exit ;; \ - *) $(call DOWNLOAD_WGET,$(1),$(2)) && exit ;; \ + git) $($(DL_MODE)_GIT) && exit ;; \ + svn) $($(DL_MODE)_SVN) && exit ;; \ + *) $(call $(DL_MODE)_WGET,$(1),$(2)) && exit ;; \ esac ; \ fi ; \ if test -n "$(call qstrip,$(BR2_BACKUP_SITE))" ; then \ - $(call DOWNLOAD_WGET,$(BR2_BACKUP_SITE),$(2)) && exit ; \ + $(call $(DL_MODE)_WGET,$(BR2_BACKUP_SITE),$(2)) && exit ; \ fi ; \ - exit 1) + exit 1 endef # Utility programs used to build packages @@ -153,14 +209,18 @@ endef # Retrieve the archive $(BUILD_DIR)/%/.stamp_downloaded: +ifeq ($(DL_MODE),DOWNLOAD) # Only show the download message if it isn't already downloaded $(Q)(test -e $(DL_DIR)/$($(PKG)_SOURCE) && \ (test -z $($(PKG)_PATCH) || test -e $(DL_DIR)$($(PKG)_PATCH))) || \ $(call MESSAGE,"Downloading") +endif $(call DOWNLOAD,$($(PKG)_SITE),$($(PKG)_SOURCE)) $(if $($(PKG)_PATCH),$(call DOWNLOAD,$($(PKG)_SITE),$($(PKG)_PATCH))) +ifeq ($(DL_MODE),DOWNLOAD) $(Q)mkdir -p $(@D) $(Q)touch $@ +endif # Unpack the archive $(BUILD_DIR)/%/.stamp_extracted: diff --git a/package/mpfr/mpfr.mk b/package/mpfr/mpfr.mk index b7b80a5a4..6979a9f76 100644 --- a/package/mpfr/mpfr.mk +++ b/package/mpfr/mpfr.mk @@ -21,8 +21,10 @@ MPFR_PATCH_SOURCE:=$(DL_DIR)/$(MPFR_PATCH_FILE) $(MPFR_PATCH_SOURCE): $(call DOWNLOAD,$(MPFR_SITE),$(MPFR_PATCH)) +ifeq ($(DL_MODE),DOWNLOAD) mv $(DL_DIR)/$(MPFR_PATCH) $@ endif +endif $(DL_DIR)/$(MPFR_SOURCE): $(call DOWNLOAD,$(MPFR_SITE),$(MPFR_SOURCE)) diff --git a/toolchain/wget-show-external-deps.sh b/toolchain/wget-show-external-deps.sh deleted file mode 100755 index 20071c05f..000000000 --- a/toolchain/wget-show-external-deps.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/sh -# -# replacement for wget (see BR2_WGET) which simply shows the file name to be -# downloaded. Used by the external-deps make target. - -exec basename ${!#} \ No newline at end of file -- cgit v1.2.3 From 1475384413f2db15bceb69c8c56454319c3ce5e7 Mon Sep 17 00:00:00 2001 From: Maxime Petazzoni Date: Thu, 2 Sep 2010 12:59:26 +0200 Subject: Revert "Makefile.package.in: allow packages to override download step" This reverts commit 48cf66f1a2e2c501abeee98b7a2268b0d6d2010d. Signed-off-by: Maxime Petazzoni Signed-off-by: Thomas Petazzoni --- package/Makefile.package.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'package/Makefile.package.in') diff --git a/package/Makefile.package.in b/package/Makefile.package.in index 6f584a747..58b417b05 100644 --- a/package/Makefile.package.in +++ b/package/Makefile.package.in @@ -402,7 +402,7 @@ $(2)_TARGET_BUILD = $$($(2)_DIR)/.stamp_built $(2)_TARGET_CONFIGURE = $$($(2)_DIR)/.stamp_configured $(2)_TARGET_PATCH = $$($(2)_DIR)/.stamp_patched $(2)_TARGET_EXTRACT = $$($(2)_DIR)/.stamp_extracted -$(2)_TARGET_SOURCE ?= $$($(2)_DIR)/.stamp_downloaded +$(2)_TARGET_SOURCE = $$($(2)_DIR)/.stamp_downloaded $(2)_TARGET_UNINSTALL = $$($(2)_DIR)/.stamp_uninstalled $(2)_TARGET_CLEAN = $$($(2)_DIR)/.stamp_cleaned $(2)_TARGET_DIRCLEAN = $$($(2)_DIR)/.stamp_dircleaned -- cgit v1.2.3