1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
|
# This class integrates real-time license scanning, generation of SPDX standard
# output and verifiying license info during the building process.
# It is a combination of efforts from the OE-Core, SPDX and fossology projects.
#
# For more information on fossology REST API:
# https://www.fossology.org/get-started/basic-rest-api-calls/
#
# For more information on SPDX:
# http://www.spdx.org
#
# Note:
# 1) Make sure fossology (after 3.5.0)(https://hub.docker.com/r/fossology/fossology/) has beed started on your host
# 2) spdx files will be output to the path which is defined as[SPDX_DEPLOY_DIR].
# By default, SPDX_DEPLOY_DIR is tmp/deploy/
# 3) Added TOKEN has been set in conf/local.conf
#
inherit spdx-common
CREATOR_TOOL = "fossdriver-host.bbclass in meta-spdxscanner"
# If ${S} isn't actually the top-level source directory, set SPDX_S to point at
# the real top-level directory.
SPDX_S ?= "${S}"
python do_spdx () {
import os, sys, json, shutil
pn = d.getVar('PN')
assume_provided = (d.getVar("ASSUME_PROVIDED") or "").split()
if pn in assume_provided:
for p in d.getVar("PROVIDES").split():
if p != pn:
pn = p
break
# glibc-locale: do_fetch, do_unpack and do_patch tasks have been deleted,
# so avoid archiving source here.
if pn.startswith('glibc-locale'):
return
if (d.getVar('PN') == "libtool-cross"):
return
if (d.getVar('PN') == "libgcc-initial"):
return
if (d.getVar('PN') == "shadow-sysroot"):
return
if d.getVar('BPN') in ['gcc', 'libgcc']:
bb.debug(1, 'spdx: There is bug in scan of %s is, do nothing' % pn)
return
# We just archive gcc-source for all the gcc related recipes
if d.getVar('BPN') in ['gcc', 'libgcc']:
bb.debug(1, 'spdx: There is bug in scan of %s is, do nothing' % pn)
return
spdx_outdir = d.getVar('SPDX_OUTDIR')
spdx_workdir = d.getVar('SPDX_WORKDIR')
spdx_temp_dir = os.path.join(spdx_workdir, "temp")
temp_dir = os.path.join(d.getVar('WORKDIR'), "temp")
info = {}
info['workdir'] = (d.getVar('WORKDIR', True) or "")
info['pn'] = (d.getVar( 'PN', True ) or "")
info['pv'] = (d.getVar( 'PV', True ) or "")
info['package_download_location'] = (d.getVar( 'SRC_URI', True ) or "")
if info['package_download_location'] != "":
info['package_download_location'] = info['package_download_location'].split()[0]
info['spdx_version'] = (d.getVar('SPDX_VERSION', True) or '')
info['data_license'] = (d.getVar('DATA_LICENSE', True) or '')
info['creator'] = {}
info['creator']['Tool'] = (d.getVar('CREATOR_TOOL', True) or '')
info['license_list_version'] = (d.getVar('LICENSELISTVERSION', True) or '')
info['package_homepage'] = (d.getVar('HOMEPAGE', True) or "")
info['package_summary'] = (d.getVar('SUMMARY', True) or "")
info['package_summary'] = info['package_summary'].replace("\n","")
info['package_summary'] = info['package_summary'].replace("'"," ")
info['package_contains'] = (d.getVar('CONTAINED', True) or "")
info['package_static_link'] = (d.getVar('STATIC_LINK', True) or "")
info['modified'] = "false"
srcuri = d.getVar("SRC_URI", False).split()
length = len("file://")
for item in srcuri:
if item.startswith("file://"):
item = item[length:]
if item.endswith(".patch") or item.endswith(".diff"):
info['modified'] = "true"
manifest_dir = (d.getVar('SPDX_DEPLOY_DIR', True) or "")
if not os.path.exists( manifest_dir ):
bb.utils.mkdirhier( manifest_dir )
info['outfile'] = os.path.join(manifest_dir, info['pn'] + "-" + info['pv'] + ".spdx" )
sstatefile = os.path.join(spdx_outdir, info['pn'] + "-" + info['pv'] + ".spdx" )
# if spdx has been exist
if os.path.exists(info['outfile']):
bb.note(info['pn'] + "spdx file has been exist, do nothing")
return
if os.path.exists( sstatefile ):
bb.note(info['pn'] + "spdx file has been exist, do nothing")
create_manifest(info,sstatefile)
return
spdx_get_src(d)
bb.note('SPDX: Archiving the patched source...')
if os.path.isdir(spdx_temp_dir):
for f_dir, f in list_files(spdx_temp_dir):
temp_file = os.path.join(spdx_temp_dir,f_dir,f)
shutil.copy(temp_file, temp_dir)
d.setVar('WORKDIR', spdx_workdir)
info['sourcedir'] = spdx_workdir
git_path = "%s/git/.git" % info['sourcedir']
if os.path.exists(git_path):
remove_dir_tree(git_path)
tar_name = spdx_create_tarball(d, d.getVar('WORKDIR'), 'patched', spdx_outdir)
## get everything from cache. use it to decide if
## something needs to be rerun
if not os.path.exists(spdx_outdir):
bb.utils.mkdirhier(spdx_outdir)
cur_ver_code = get_ver_code(spdx_workdir).split()[0]
## Get spdx file
bb.note(' run fossdriver ...... ')
if not os.path.isfile(tar_name):
bb.warn(info['pn'] + "has no source, do nothing")
return
invoke_fossdriver(tar_name,sstatefile)
if get_cached_spdx(sstatefile) != None:
write_cached_spdx( info,sstatefile,cur_ver_code )
## CREATE MANIFEST(write to outfile )
create_manifest(info,sstatefile)
else:
bb.warn('Can\'t get the spdx file ' + info['pn'] + '. Please check your.')
remove_file(tar_name)
}
def invoke_fossdriver(tar_file, spdx_file):
import os
import time
delaytime = 20
import logging
logger = logging.getLogger()
logger.setLevel(logging.INFO)
logging.basicConfig(level=logging.INFO)
(work_dir, tar_file) = os.path.split(tar_file)
os.chdir(work_dir)
from fossdriver.config import FossConfig
from fossdriver.server import FossServer
from fossdriver.tasks import (CreateFolder, Upload, Scanners, Copyright, Reuse, BulkTextMatch, SPDXTV)
if 'http_proxy' in os.environ:
del os.environ['http_proxy']
config = FossConfig()
configPath = os.path.join(os.path.expanduser('~'),".fossdriverrc")
config.configure(configPath)
server = FossServer(config)
server.Login()
bb.note("invoke_fossdriver : tar_file = %s " % tar_file)
if (Reuse(server, tar_file, "Software Repository", tar_file, "Software Repository").run() != True):
bb.note("This OSS has not been scanned. So upload it to fossology server.")
i = 0
while i < 5:
if (Upload(server, tar_file, "Software Repository").run() != True):
bb.warn("%s Upload failed, try again!" % tar_file)
time.sleep(delaytime)
i += 1
else:
i = 0
while i < 10:
if (Scanners(server, tar_file, "Software Repository").run() != True):
bb.warn("%s Scanners failed, try again!" % tar_file)
time.sleep(delaytime)
i+= 1
else:
i = 0
while i < 10:
Copyright(server, tar_file, "Software Repository").run()
if (SPDXTV(server, tar_file, "Software Repository", spdx_file).run() == False):
bb.warn("%s SPDXTV failed, try again!" % tar_file)
time.sleep(delaytime)
i += 1
else:
return True
bb.warn("%s SPDXTV failed, Please check your fossology server." % tar_file)
return False
bb.warn("%s Scanners failed, Please check your fossology server." % tar_file)
return False
bb.warn("%s Upload fail.Please check your fossology server." % tar_file)
return False
else:
i = 0
while i < 10:
if (SPDXTV(server, tar_file, "Software Repository", spdx_file).run() == False):
time.sleep(1)
bb.warn("%s SPDXTV failed, try again!" % tar_file)
i += 1
time.sleep(delaytime)
else:
return True
bb.warn("%s SPDXTV failed, Please check your fossology server." % tar_file)
return False
EXPORT_FUNCTIONS do_spdx
|