Skip to content

Commit f848741

Browse files
AyanSinhaMahapatraAyanSinhaMahapatra
committed
Add attribute to track private packages #3102
Reference: #3102 Reference: #1514 Signed-off-by: Ayan Sinha Mahapatra <ayansmahapatra@gmail.com>
1 parent 86d9d63 commit f848741

File tree

1,141 files changed

+12063
-3601
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

1,141 files changed

+12063
-3601
lines changed

‎src/packagedcode/bower.py

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,10 @@ def parse(cls, location, package_only=False):
2929
with io.open(location, encoding='utf-8') as loc:
3030
package_data = json.load(loc)
3131

32-
# note: having no name is not a problem for private packages. See #1514
3332
name = package_data.get('name')
33+
is_private = False
34+
if not name:
35+
is_private = True
3436

3537
description = package_data.get('description')
3638
version = package_data.get('version')
@@ -99,5 +101,6 @@ def parse(cls, location, package_only=False):
99101
homepage_url=homepage_url,
100102
vcs_url=vcs_url,
101103
dependencies=dependencies,
104+
is_private=is_private,
102105
)
103106
yield models.PackageData.from_data(package_data, package_only)

‎src/packagedcode/models.py

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -682,6 +682,14 @@ class PackageData(IdentifiablePackageData):
682682
'package type or datafile format.'
683683
)
684684

685+
is_private = Boolean(
686+
default=False,
687+
label='is resolved flag',
688+
help='True if the associated package for this package manifest '
689+
'is never meant to be published to the corresponding package '
690+
'repository, and is a private package.'
691+
)
692+
685693
extra_data = Mapping(
686694
label='extra data',
687695
help='A mapping of arbitrary extra package data.',

‎src/packagedcode/npm.py

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -212,7 +212,17 @@ def update_dependencies_by_purl(
212212

213213
if scope in metadata_deps :
214214
dep_package = dependecies_by_purl.get(dep_purl)
215-
dep_package.is_optional = metadata.get("optional")
215+
if dep_package:
216+
dep_package.is_optional = metadata.get("optional")
217+
else:
218+
dep_package = models.DependentPackage(
219+
purl=dep_purl,
220+
scope=scope,
221+
is_runtime=is_runtime,
222+
is_optional=metadata.get("optional"),
223+
is_resolved=is_resolved,
224+
)
225+
dependecies_by_purl[dep_purl] = dep_package
216226
continue
217227

218228
# pnpm has peer dependencies also sometimes in version?
@@ -266,7 +276,11 @@ def _parse(cls, json_data, package_only=False):
266276

267277
namespace, name = split_scoped_package_name(name)
268278

269-
urls = get_urls(namespace, name, version)
279+
is_private = json_data.get('private') or False
280+
if is_private:
281+
urls = {}
282+
else:
283+
urls = get_urls(namespace, name, version)
270284
package_data = dict(
271285
datasource_id=cls.datasource_id,
272286
type=cls.default_package_type,
@@ -276,6 +290,7 @@ def _parse(cls, json_data, package_only=False):
276290
version=version or None,
277291
description=json_data.get('description', '').strip() or None,
278292
homepage_url=homepage_url,
293+
is_private=is_private,
279294
**urls,
280295
)
281296
package = models.PackageData.from_data(package_data, package_only)

‎src/packagedcode/phpcomposer.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,7 @@ def build_package_data(package_data, package_only=False):
111111
repository_homepage_url=get_repository_homepage_url(ns, name),
112112
api_data_url=get_api_data_url(ns, name),
113113
primary_language=PhpComposerJsonHandler.default_primary_language,
114+
is_private=is_private,
114115
)
115116
package = models.PackageData.from_data(package_mapping, package_only)
116117

‎tests/formattedcode/data/common/manifests-expected.json

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,7 @@
7373
"source_packages": [
7474
"pkg:maven/javax.persistence/persistence-api@1.0?classifier=sources"
7575
],
76+
"is_private": false,
7677
"extra_data": {},
7778
"repository_homepage_url": "https://repo1.maven.org/maven2/javax/persistence/persistence-api/1.0/",
7879
"repository_download_url": "https://repo1.maven.org/maven2/javax/persistence/persistence-api/1.0/persistence-api-1.0.jar",
@@ -174,6 +175,7 @@
174175
"extracted_license_statement": "- Apache-2.0\n- type: Apache 2.0\n url: https://github.com/spenceralger/grunt-esvm/blob/master/LICENSE.md\n",
175176
"notice_text": null,
176177
"source_packages": [],
178+
"is_private": false,
177179
"extra_data": {},
178180
"repository_homepage_url": "https://www.npmjs.com/package/grunt-esvm",
179181
"repository_download_url": "https://registry.npmjs.org/grunt-esvm/-/grunt-esvm-3.2.8.tgz",
@@ -251,6 +253,7 @@
251253
"extracted_license_statement": "- MIT\n",
252254
"notice_text": null,
253255
"source_packages": [],
256+
"is_private": false,
254257
"extra_data": {},
255258
"repository_homepage_url": "https://www.npmjs.com/package/angular-compare-validator",
256259
"repository_download_url": "https://registry.npmjs.org/angular-compare-validator/-/angular-compare-validator-0.1.1.tgz",
@@ -892,6 +895,7 @@
892895
"pkg:maven/javax.persistence/persistence-api@1.0?classifier=sources"
893896
],
894897
"file_references": [],
898+
"is_private": false,
895899
"extra_data": {},
896900
"dependencies": [],
897901
"repository_homepage_url": "https://repo1.maven.org/maven2/javax/persistence/persistence-api/1.0/",
@@ -1096,6 +1100,7 @@
10961100
"notice_text": null,
10971101
"source_packages": [],
10981102
"file_references": [],
1103+
"is_private": false,
10991104
"extra_data": {},
11001105
"dependencies": [
11011106
{
@@ -1390,6 +1395,7 @@
13901395
"notice_text": null,
13911396
"source_packages": [],
13921397
"file_references": [],
1398+
"is_private": false,
13931399
"extra_data": {},
13941400
"dependencies": [
13951401
{
@@ -1639,6 +1645,7 @@
16391645
"notice_text": null,
16401646
"source_packages": [],
16411647
"file_references": [],
1648+
"is_private": false,
16421649
"extra_data": {},
16431650
"dependencies": [
16441651
{

‎tests/formattedcode/data/common/manifests-expected.jsonlines

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -12,19 +12,19 @@
1212
"--package": true
1313
},
1414
"notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.",
15-
"output_format_version": "3.0.0",
15+
"output_format_version": "3.1.0",
1616
"message": null,
1717
"errors": [],
1818
"warnings": [],
1919
"extra_data": {
2020
"system_environment": {
2121
"operating_system": "linux",
2222
"cpu_architecture": "64",
23-
"platform": "Linux-5.15.0-89-generic-x86_64-with-glibc2.29",
24-
"platform_version": "#99~20.04.1-Ubuntu SMP Thu Nov 2 15:16:47 UTC 2023",
25-
"python_version": "3.8.10 (default, Nov 22 2023, 10:22:35) \n[GCC 9.4.0]"
23+
"platform": "Linux-5.15.0-106-generic-x86_64-with-glibc2.35",
24+
"platform_version": "#116-Ubuntu SMP Wed Apr 17 09:17:56 UTC 2024",
25+
"python_version": "3.10.12 (main, Nov 20 2023, 15:14:05) [GCC 11.4.0]"
2626
},
27-
"spdx_license_list_version": "3.22",
27+
"spdx_license_list_version": "3.23",
2828
"files_count": 4
2929
}
3030
}
@@ -105,6 +105,7 @@
105105
"source_packages": [
106106
"pkg:maven/javax.persistence/persistence-api@1.0?classifier=sources"
107107
],
108+
"is_private": false,
108109
"extra_data": {},
109110
"repository_homepage_url": "https://repo1.maven.org/maven2/javax/persistence/persistence-api/1.0/",
110111
"repository_download_url": "https://repo1.maven.org/maven2/javax/persistence/persistence-api/1.0/persistence-api-1.0.jar",
@@ -206,6 +207,7 @@
206207
"extracted_license_statement": "- Apache-2.0\n- type: Apache 2.0\n url: https://github.com/spenceralger/grunt-esvm/blob/master/LICENSE.md\n",
207208
"notice_text": null,
208209
"source_packages": [],
210+
"is_private": false,
209211
"extra_data": {},
210212
"repository_homepage_url": "https://www.npmjs.com/package/grunt-esvm",
211213
"repository_download_url": "https://registry.npmjs.org/grunt-esvm/-/grunt-esvm-3.2.8.tgz",
@@ -283,6 +285,7 @@
283285
"extracted_license_statement": "- MIT\n",
284286
"notice_text": null,
285287
"source_packages": [],
288+
"is_private": false,
286289
"extra_data": {},
287290
"repository_homepage_url": "https://www.npmjs.com/package/angular-compare-validator",
288291
"repository_download_url": "https://registry.npmjs.org/angular-compare-validator/-/angular-compare-validator-0.1.1.tgz",
@@ -938,6 +941,7 @@
938941
"pkg:maven/javax.persistence/persistence-api@1.0?classifier=sources"
939942
],
940943
"file_references": [],
944+
"is_private": false,
941945
"extra_data": {},
942946
"dependencies": [],
943947
"repository_homepage_url": "https://repo1.maven.org/maven2/javax/persistence/persistence-api/1.0/",
@@ -1150,6 +1154,7 @@
11501154
"notice_text": null,
11511155
"source_packages": [],
11521156
"file_references": [],
1157+
"is_private": false,
11531158
"extra_data": {},
11541159
"dependencies": [
11551160
{
@@ -1452,6 +1457,7 @@
14521457
"notice_text": null,
14531458
"source_packages": [],
14541459
"file_references": [],
1460+
"is_private": false,
14551461
"extra_data": {},
14561462
"dependencies": [
14571463
{
@@ -1709,6 +1715,7 @@
17091715
"notice_text": null,
17101716
"source_packages": [],
17111717
"file_references": [],
1718+
"is_private": false,
17121719
"extra_data": {},
17131720
"dependencies": [
17141721
{

‎tests/formattedcode/data/common/manifests-expected.yaml

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -21,18 +21,18 @@ headers:
2121
for any legal advice.
2222
ScanCode is a free software code scanning tool from nexB Inc. and others.
2323
Visit https://github.com/nexB/scancode-toolkit/ for support and download.
24-
output_format_version: 3.0.0
24+
output_format_version: 3.1.0
2525
message:
2626
errors: []
2727
warnings: []
2828
extra_data:
2929
system_environment:
3030
operating_system: linux
3131
cpu_architecture: 64
32-
platform: Linux-5.15.0-89-generic-x86_64-with-glibc2.29
33-
platform_version: '#99~20.04.1-Ubuntu SMP Thu Nov 2 15:16:47 UTC 2023'
34-
python_version: "3.8.10 (default, Nov 22 2023, 10:22:35) \n[GCC 9.4.0]"
35-
spdx_license_list_version: '3.22'
32+
platform: Linux-5.15.0-106-generic-x86_64-with-glibc2.35
33+
platform_version: '#116-Ubuntu SMP Wed Apr 17 09:17:56 UTC 2024'
34+
python_version: 3.10.12 (main, Nov 20 2023, 15:14:05) [GCC 11.4.0]
35+
spdx_license_list_version: '3.23'
3636
files_count: 4
3737
summary:
3838
declared_license_expression: apache-2.0 AND cddl-1.0 AND mit
@@ -130,6 +130,7 @@ packages:
130130
notice_text:
131131
source_packages:
132132
- pkg:maven/javax.persistence/persistence-api@1.0?classifier=sources
133+
is_private: no
133134
extra_data: {}
134135
repository_homepage_url: https://repo1.maven.org/maven2/javax/persistence/persistence-api/1.0/
135136
repository_download_url: https://repo1.maven.org/maven2/javax/persistence/persistence-api/1.0/persistence-api-1.0.jar
@@ -215,6 +216,7 @@ packages:
215216
url: https://github.com/spenceralger/grunt-esvm/blob/master/LICENSE.md
216217
notice_text:
217218
source_packages: []
219+
is_private: no
218220
extra_data: {}
219221
repository_homepage_url: https://www.npmjs.com/package/grunt-esvm
220222
repository_download_url: https://registry.npmjs.org/grunt-esvm/-/grunt-esvm-3.2.8.tgz
@@ -280,6 +282,7 @@ packages:
280282
- MIT
281283
notice_text:
282284
source_packages: []
285+
is_private: no
283286
extra_data: {}
284287
repository_homepage_url: https://www.npmjs.com/package/angular-compare-validator
285288
repository_download_url: https://registry.npmjs.org/angular-compare-validator/-/angular-compare-validator-0.1.1.tgz
@@ -1791,6 +1794,7 @@ files:
17911794
source_packages:
17921795
- pkg:maven/javax.persistence/persistence-api@1.0?classifier=sources
17931796
file_references: []
1797+
is_private: no
17941798
extra_data: {}
17951799
dependencies: []
17961800
repository_homepage_url: https://repo1.maven.org/maven2/javax/persistence/persistence-api/1.0/
@@ -1995,6 +1999,7 @@ files:
19951999
notice_text:
19962000
source_packages: []
19972001
file_references: []
2002+
is_private: no
19982003
extra_data: {}
19992004
dependencies:
20002005
- purl: pkg:npm/bluebird
@@ -2269,6 +2274,7 @@ files:
22692274
notice_text:
22702275
source_packages: []
22712276
file_references: []
2277+
is_private: no
22722278
extra_data: {}
22732279
dependencies:
22742280
- purl: pkg:npm/%40angular/core
@@ -2512,6 +2518,7 @@ files:
25122518
notice_text:
25132519
source_packages: []
25142520
file_references: []
2521+
is_private: no
25152522
extra_data: {}
25162523
dependencies:
25172524
- purl: pkg:pypi/numpy

0 commit comments

Comments
 (0)