1#!/usr/bin/env python
2#
3# Copyright (c) nexB Inc. and others. All rights reserved.
4# ScanCode is a trademark of nexB Inc.
5# SPDX-License-Identifier: Apache-2.0
6# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
7# See https://github.com/aboutcode-org/skeleton for support or download.
8# See https://aboutcode.org for more information about nexB OSS projects.
…lines 9–20 unchanged (12 lines)
21
22import attr
23import license_expression
24import packageurl
25import requests
26import saneyaml
27import utils_pip_compatibility_tags
28from commoncode import fileutils
29from commoncode.hash import multi_checksums
30from commoncode.text import python_safe_name
31from packvers import tags as packaging_tags
32from packvers import version as packaging_version
33
…lines 34–87 unchanged (54 lines)
88
89"""
90Wheel downloader
91
92- parse requirement file
93- create a TODO queue of requirements to process
94- done: create an empty map of processed binary requirements as
95 {package name: (list of versions/tags}
96
97
98- while we have package reqs in TODO queue, process one requirement:
99 - for each PyPI simple index:
100 - fetch through cache the PyPI simple index for this package
101 - for each environment:
…lines 102–110 unchanged (9 lines)
111
112TRACE = False
113TRACE_DEEP = False
114TRACE_ULTRA_DEEP = False
115
116# Supported environments
117PYTHON_VERSIONS = "37", "38", "39", "310"
118
119PYTHON_DOT_VERSIONS_BY_VER = {
120 "37": "3.7",
121 "38": "3.8",
122 "39": "3.9",
123 "310": "3.10",
124}
125
126
127def get_python_dot_version(version):
128 """
129 Return a dot version from a plain, non-dot version.
130 """
131 return PYTHON_DOT_VERSIONS_BY_VER[version]
132
133
134ABIS_BY_PYTHON_VERSION = {
135 "37": ["cp37", "cp37m", "abi3"],
136 "38": ["cp38", "cp38m", "abi3"],
137 "39": ["cp39", "cp39m", "abi3"],
138 "310": ["cp310", "cp310m", "abi3"],
139}
140
141PLATFORMS_BY_OS = {
142 "linux": [
143 "linux_x86_64",
144 "manylinux1_x86_64",
145 "manylinux2010_x86_64",
…lines 146–549 unchanged (404 lines)
550
551 def download(self, dest_dir=THIRDPARTY_DIR):
552 """
553 Download this distribution into `dest_dir` directory.
554 Return the fetched filename.
555 """
556 if not self.filename:
557 raise ValueError(f"self.filename has no value but is required: {self.filename!r}")
558 if TRACE_DEEP:
559 print(
560 f"Fetching distribution of {self.name}=={self.version}:",
561 self.filename,
562 )
563
…lines 564–818 unchanged (255 lines)
819 """
820 Fetch license files if missing in `dest_dir`.
821 Return True if license files were fetched.
822 """
823 urls = LinksRepository.from_url(use_cached_index=use_cached_index).links
824 errors = []
825 extra_lic_names = [lic.get("file") for lic in self.extra_data.get("licenses", {})]
826 extra_lic_names += [self.extra_data.get("license_file")]
827 extra_lic_names = [eln for eln in extra_lic_names if eln]
828 lic_names = [f"{key}.LICENSE" for key in self.get_license_keys()]
829 for filename in lic_names + extra_lic_names:
830 floc = os.path.join(dest_dir, filename)
831 if os.path.exists(floc):
832 continue
833
…lines 834–840 unchanged (7 lines)
841 filename=filename,
842 as_text=True,
843 )
844 if TRACE:
845 print(f"Fetched license from remote: {lic_url}")
846
847 except Exception:
848 try:
849 # try licensedb second
850 lic_url = f"{LICENSEDB_API_URL}/{filename}"
851 fetch_and_save(
852 path_or_url=lic_url,
853 dest_dir=dest_dir,
854 filename=filename,
855 as_text=True,
856 )
857 if TRACE:
858 print(f"Fetched license from licensedb: {lic_url}")
859
860 except Exception:
861 msg = f"No text for license {filename} in expression "
862 f"{self.license_expression!r} from {self}"
863 print(msg)
864 errors.append(msg)
865
866 return errors
867
868 def extract_pkginfo(self, dest_dir=THIRDPARTY_DIR):
…lines 869–995 unchanged (127 lines)
996def get_license_link_for_filename(filename, urls):
997 """
998 Return a link for `filename` found in the `links` list of URLs or paths. Raise an
999 exception if no link is found or if there are more than one link for that
1000 file name.
1001 """
1002 path_or_url = [url for url in urls if url.endswith(f"/{filename}")]
1003 if not path_or_url:
1004 raise Exception(f"Missing link to file: {filename}")
1005 if not len(path_or_url) == 1:
1006 raise Exception(f"Multiple links to file: {filename}: \n" + "\n".join(path_or_url))
1007 return path_or_url[0]
1008
…lines 1009–1284 unchanged (276 lines)
1285 return "py3" in self.python_versions and "none" in self.abis and "any" in self.platforms
1286
1287
1288def is_pure_wheel(filename):
1289 try:
1290 return Wheel.from_filename(filename).is_pure()
1291 except Exception:
1292 return False
1293
1294
1295@attr.attributes
1296class PypiPackage(NameVer):
1297 """
…lines 1298–1480 unchanged (183 lines)
1481 def get_distributions(self):
1482 """
1483 Yield all distributions available for this PypiPackage
1484 """
1485 if self.sdist:
1486 yield self.sdist
1487 yield from self.wheels
1488
1489 def get_url_for_filename(self, filename):
1490 """
1491 Return the URL for this filename or None.
1492 """
1493 for dist in self.get_distributions():
…lines 1494–1609 unchanged (116 lines)
1610 # the inner versions mapping is sorted by version from oldest to newest
1611
1612 packages = attr.ib(
1613 type=dict,
1614 default=attr.Factory(lambda: defaultdict(dict)),
1615 metadata=dict(
1616 help="Mapping of {name: {version: PypiPackage, version: PypiPackage, etc} "
1617 "available in this repo"
1618 ),
1619 )
1620
1621 fetched_package_normalized_names = attr.ib(
1622 type=set,
1623 default=attr.Factory(set),
1624 metadata=dict(help="A set of already fetched package normalized names."),
1625 )
1626
1627 use_cached_index = attr.ib(
1628 type=bool,
1629 default=False,
1630 metadata=dict(
1631 help="If True, use any existing on-disk cached PyPI index files. "
1632 "Otherwise, fetch and cache."
1633 ),
1634 )
1635
1636 def _get_package_versions_map(self, name):
1637 """
1638 Return a mapping of all available PypiPackage version for this package name.
1639 The mapping may be empty. It is ordered by version from oldest to newest
1640 """
1641 if not name:
1642 raise ValueError(f"name is required: {name!r}")
1643 normalized_name = NameVer.normalize_name(name)
1644 versions = self.packages[normalized_name]
1645 if not versions and normalized_name not in self.fetched_package_normalized_names:
1646 self.fetched_package_normalized_names.add(normalized_name)
1647 try:
1648 links = self.fetch_links(normalized_name=normalized_name)
…lines 1649–1690 unchanged (42 lines)
1691 path_or_url=package_url,
1692 as_text=True,
1693 force=not self.use_cached_index,
1694 )
1695 links = collect_urls(text)
1696 # TODO: keep sha256
1697 links = [link.partition("#sha256=") for link in links]
1698 links = [url for url, _, _sha256 in links]
1699 return links
1700
1701
1702PYPI_PUBLIC_REPO = PypiSimpleRepository(index_url=PYPI_SIMPLE_URL)
1703PYPI_SELFHOSTED_REPO = PypiSimpleRepository(index_url=ABOUT_PYPI_SIMPLE_URL)
…lines 1704–1911 unchanged (208 lines)
1912 time.sleep(_delay)
1913 headers = headers or {}
1914 # using a GET with stream=True ensure we get the the final header from
1915 # several redirects and that we can ignore content there. A HEAD request may
1916 # not get us this last header
1917 print(f" DOWNLOADING: {url}")
1918 with requests.get(url, allow_redirects=True, stream=True, headers=headers) as response: # noqa: S113
1919 status = response.status_code
1920 if status != requests.codes.ok: # NOQA
1921 if status == 429 and _delay < 20:
1922 # too many requests: start some exponential delay
1923 increased_delay = (_delay * 2) or 1
1924
…lines 1925–2130 unchanged (206 lines)
2131 """
2132 Call args in a subprocess and display output on the fly if ``trace`` is True.
2133 Return a tuple of (returncode, stdout, stderr)
2134 """
2135 if TRACE_DEEP:
2136 print("Calling:", " ".join(args))
2137 with subprocess.Popen( # noqa: S603
2138 args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding="utf-8"
2139 ) as process:
2140 stdouts = []
2141 while True:
2142 line = process.stdout.readline()
2143 if not line and process.poll() is not None:
…lines 2144–2195 unchanged (52 lines)
2196
2197 cli_args.extend(requirements_specifiers)
2198 for req_file in requirements_files:
2199 cli_args.extend(["--requirement", req_file])
2200
2201 if TRACE:
2202 print("Downloading wheels using command:", " ".join(cli_args))
2203
2204 existing = set(os.listdir(dest_dir))
2205 error = False
2206 try:
2207 returncode, _stdout, stderr = call(cli_args, verbose=True)
2208 if returncode != 0:
…lines 2209–2228 unchanged (20 lines)
2229#
2230################################################################################
2231
2232
2233def check_about(dest_dir=THIRDPARTY_DIR):
2234 try:
2235 subprocess.check_output(f"venv/bin/about check {dest_dir}".split()) # noqa: S603
2236 except subprocess.CalledProcessError as cpe:
2237 print()
2238 print("Invalid ABOUT files:")
2239 print(cpe.output.decode("utf-8", errors="replace"))
2240
2241
…lines 2242–2279 unchanged (38 lines)
2280 try:
2281 from packagedcode.licensing import get_only_expression_from_extracted_license
2282
2283 return get_only_expression_from_extracted_license(declared_licenses)
2284 except ImportError:
2285 # Scancode is not installed, clean and join all the licenses
2286 lics = [python_safe_name(lic).lower() for lic in declared_licenses]
2287 return " AND ".join(lics).lower()
2288