Branch data Line data Source code
1 : : /* Debuginfo-over-http server.
2 : : Copyright (C) 2019-2021 Red Hat, Inc.
3 : : Copyright (C) 2021, 2022 Mark J. Wielaard <mark@klomp.org>
4 : : This file is part of elfutils.
5 : :
6 : : This file is free software; you can redistribute it and/or modify
7 : : it under the terms of the GNU General Public License as published by
8 : : the Free Software Foundation; either version 3 of the License, or
9 : : (at your option) any later version.
10 : :
11 : : elfutils is distributed in the hope that it will be useful, but
12 : : WITHOUT ANY WARRANTY; without even the implied warranty of
13 : : MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 : : GNU General Public License for more details.
15 : :
16 : : You should have received a copy of the GNU General Public License
17 : : along with this program. If not, see <http://www.gnu.org/licenses/>. */
18 : :
19 : :
20 : : /* cargo-cult from libdwfl linux-kernel-modules.c */
21 : : /* In case we have a bad fts we include this before config.h because it
22 : : can't handle _FILE_OFFSET_BITS.
23 : : Everything we need here is fine if its declarations just come first.
24 : : Also, include sys/types.h before fts. On some systems fts.h is not self
25 : : contained. */
26 : : #ifdef BAD_FTS
27 : : #include <sys/types.h>
28 : : #include <fts.h>
29 : : #endif
30 : :
31 : : #ifdef HAVE_CONFIG_H
32 : : #include "config.h"
33 : : #endif
34 : :
35 : : // #define _GNU_SOURCE
36 : : #ifdef HAVE_SCHED_H
37 : : extern "C" {
38 : : #include <sched.h>
39 : : }
40 : : #endif
41 : : #ifdef HAVE_SYS_RESOURCE_H
42 : : extern "C" {
43 : : #include <sys/resource.h>
44 : : }
45 : : #endif
46 : :
47 : : #ifdef HAVE_EXECINFO_H
48 : : extern "C" {
49 : : #include <execinfo.h>
50 : : }
51 : : #endif
52 : :
53 : : extern "C" {
54 : : #include "printversion.h"
55 : : #include "system.h"
56 : : }
57 : :
58 : : #include "debuginfod.h"
59 : : #include <dwarf.h>
60 : :
61 : : #include <argp.h>
62 : : #ifdef __GNUC__
63 : : #undef __attribute__ /* glibc bug - rhbz 1763325 */
64 : : #endif
65 : :
66 : : #include <unistd.h>
67 : : #include <stdlib.h>
68 : : #include <locale.h>
69 : : #include <pthread.h>
70 : : #include <signal.h>
71 : : #include <sys/stat.h>
72 : : #include <sys/time.h>
73 : : #include <sys/vfs.h>
74 : : #include <unistd.h>
75 : : #include <fcntl.h>
76 : : #include <netdb.h>
77 : :
78 : :
79 : : /* If fts.h is included before config.h, its indirect inclusions may not
80 : : give us the right LFS aliases of these functions, so map them manually. */
81 : : #ifdef BAD_FTS
82 : : #ifdef _FILE_OFFSET_BITS
83 : : #define open open64
84 : : #define fopen fopen64
85 : : #endif
86 : : #else
87 : : #include <sys/types.h>
88 : : #include <fts.h>
89 : : #endif
90 : :
91 : : #include <cstring>
92 : : #include <vector>
93 : : #include <set>
94 : : #include <unordered_set>
95 : : #include <map>
96 : : #include <string>
97 : : #include <iostream>
98 : : #include <iomanip>
99 : : #include <ostream>
100 : : #include <sstream>
101 : : #include <mutex>
102 : : #include <deque>
103 : : #include <condition_variable>
104 : : #include <exception>
105 : : #include <thread>
106 : : // #include <regex> // on rhel7 gcc 4.8, not competent
107 : : #include <regex.h>
108 : : // #include <algorithm>
109 : : using namespace std;
110 : :
111 : : #include <gelf.h>
112 : : #include <libdwelf.h>
113 : :
114 : : #include <microhttpd.h>
115 : :
116 : : #if MHD_VERSION >= 0x00097002
117 : : // libmicrohttpd 0.9.71 broke API
118 : : #define MHD_RESULT enum MHD_Result
119 : : #else
120 : : #define MHD_RESULT int
121 : : #endif
122 : :
123 : : #include <curl/curl.h>
124 : : #include <archive.h>
125 : : #include <archive_entry.h>
126 : : #include <sqlite3.h>
127 : :
128 : : #ifdef __linux__
129 : : #include <sys/syscall.h>
130 : : #endif
131 : :
132 : : #ifdef __linux__
133 : : #define tid() syscall(SYS_gettid)
134 : : #else
135 : : #define tid() pthread_self()
136 : : #endif
137 : :
138 : :
139 : : inline bool
140 : 3033 : string_endswith(const string& haystack, const string& needle)
141 : : {
142 [ + - ]: 3033 : return (haystack.size() >= needle.size() &&
143 : 3033 : equal(haystack.end()-needle.size(), haystack.end(),
144 : 3033 : needle.begin()));
145 : : }
146 : :
147 : :
148 : : // Roll this identifier for every sqlite schema incompatibility.
149 : : #define BUILDIDS "buildids10"
150 : :
151 : : #if SQLITE_VERSION_NUMBER >= 3008000
152 : : #define WITHOUT_ROWID "without rowid"
153 : : #else
154 : : #define WITHOUT_ROWID ""
155 : : #endif
156 : :
157 : : static const char DEBUGINFOD_SQLITE_DDL[] =
158 : : "pragma foreign_keys = on;\n"
159 : : "pragma synchronous = 0;\n" // disable fsync()s - this cache is disposable across a machine crash
160 : : "pragma journal_mode = wal;\n" // https://sqlite.org/wal.html
161 : : "pragma wal_checkpoint = truncate;\n" // clean out any preexisting wal file
162 : : "pragma journal_size_limit = 0;\n" // limit steady state file (between grooming, which also =truncate's)
163 : : "pragma auto_vacuum = incremental;\n" // https://sqlite.org/pragma.html
164 : : "pragma busy_timeout = 1000;\n" // https://sqlite.org/pragma.html
165 : : // NB: all these are overridable with -D option
166 : :
167 : : // Normalization table for interning file names
168 : : "create table if not exists " BUILDIDS "_fileparts (\n"
169 : : " id integer primary key not null,\n"
170 : : " name text unique not null\n"
171 : : " );\n"
172 : : "create table if not exists " BUILDIDS "_files (\n"
173 : : " id integer primary key not null,\n"
174 : : " dirname integer not null,\n"
175 : : " basename integer not null,\n"
176 : : " unique (dirname, basename),\n"
177 : : " foreign key (dirname) references " BUILDIDS "_fileparts(id) on delete cascade,\n"
178 : : " foreign key (basename) references " BUILDIDS "_fileparts(id) on delete cascade\n"
179 : : " );\n"
180 : : "create view if not exists " BUILDIDS "_files_v as\n" // a
181 : : " select f.id, n1.name || '/' || n2.name as name\n"
182 : : " from " BUILDIDS "_files f, " BUILDIDS "_fileparts n1, " BUILDIDS "_fileparts n2\n"
183 : : " where f.dirname = n1.id and f.basename = n2.id;\n"
184 : :
185 : : // Normalization table for interning buildids
186 : : "create table if not exists " BUILDIDS "_buildids (\n"
187 : : " id integer primary key not null,\n"
188 : : " hex text unique not null);\n"
189 : : // Track the completion of scanning of a given file & sourcetype at given time
190 : : "create table if not exists " BUILDIDS "_file_mtime_scanned (\n"
191 : : " mtime integer not null,\n"
192 : : " file integer not null,\n"
193 : : " size integer not null,\n" // in bytes
194 : : " sourcetype text(1) not null\n"
195 : : " check (sourcetype IN ('F', 'R')),\n"
196 : : " foreign key (file) references " BUILDIDS "_files(id) on update cascade on delete cascade,\n"
197 : : " primary key (file, mtime, sourcetype)\n"
198 : : " ) " WITHOUT_ROWID ";\n"
199 : : "create table if not exists " BUILDIDS "_f_de (\n"
200 : : " buildid integer not null,\n"
201 : : " debuginfo_p integer not null,\n"
202 : : " executable_p integer not null,\n"
203 : : " file integer not null,\n"
204 : : " mtime integer not null,\n"
205 : : " foreign key (file) references " BUILDIDS "_files(id) on update cascade on delete cascade,\n"
206 : : " foreign key (buildid) references " BUILDIDS "_buildids(id) on update cascade on delete cascade,\n"
207 : : " primary key (buildid, file, mtime)\n"
208 : : " ) " WITHOUT_ROWID ";\n"
209 : : // Index for faster delete by file identifier
210 : : "create index if not exists " BUILDIDS "_f_de_idx on " BUILDIDS "_f_de (file, mtime);\n"
211 : : "create table if not exists " BUILDIDS "_f_s (\n"
212 : : " buildid integer not null,\n"
213 : : " artifactsrc integer not null,\n"
214 : : " file integer not null,\n" // NB: not necessarily entered into _mtime_scanned
215 : : " mtime integer not null,\n"
216 : : " foreign key (file) references " BUILDIDS "_files(id) on update cascade on delete cascade,\n"
217 : : " foreign key (artifactsrc) references " BUILDIDS "_files(id) on update cascade on delete cascade,\n"
218 : : " foreign key (buildid) references " BUILDIDS "_buildids(id) on update cascade on delete cascade,\n"
219 : : " primary key (buildid, artifactsrc, file, mtime)\n"
220 : : " ) " WITHOUT_ROWID ";\n"
221 : : "create table if not exists " BUILDIDS "_r_de (\n"
222 : : " buildid integer not null,\n"
223 : : " debuginfo_p integer not null,\n"
224 : : " executable_p integer not null,\n"
225 : : " file integer not null,\n"
226 : : " mtime integer not null,\n"
227 : : " content integer not null,\n"
228 : : " foreign key (file) references " BUILDIDS "_files(id) on update cascade on delete cascade,\n"
229 : : " foreign key (content) references " BUILDIDS "_files(id) on update cascade on delete cascade,\n"
230 : : " foreign key (buildid) references " BUILDIDS "_buildids(id) on update cascade on delete cascade,\n"
231 : : " primary key (buildid, debuginfo_p, executable_p, file, content, mtime)\n"
232 : : " ) " WITHOUT_ROWID ";\n"
233 : : // Index for faster delete by archive file identifier
234 : : "create index if not exists " BUILDIDS "_r_de_idx on " BUILDIDS "_r_de (file, mtime);\n"
235 : : "create table if not exists " BUILDIDS "_r_sref (\n" // outgoing dwarf sourcefile references from rpm
236 : : " buildid integer not null,\n"
237 : : " artifactsrc integer not null,\n"
238 : : " foreign key (artifactsrc) references " BUILDIDS "_files(id) on update cascade on delete cascade,\n"
239 : : " foreign key (buildid) references " BUILDIDS "_buildids(id) on update cascade on delete cascade,\n"
240 : : " primary key (buildid, artifactsrc)\n"
241 : : " ) " WITHOUT_ROWID ";\n"
242 : : "create table if not exists " BUILDIDS "_r_sdef (\n" // rpm contents that may satisfy sref
243 : : " file integer not null,\n"
244 : : " mtime integer not null,\n"
245 : : " content integer not null,\n"
246 : : " foreign key (file) references " BUILDIDS "_files(id) on update cascade on delete cascade,\n"
247 : : " foreign key (content) references " BUILDIDS "_files(id) on update cascade on delete cascade,\n"
248 : : " primary key (content, file, mtime)\n"
249 : : " ) " WITHOUT_ROWID ";\n"
250 : : // create views to glue together some of the above tables, for webapi D queries
251 : : "create view if not exists " BUILDIDS "_query_d as \n"
252 : : "select\n"
253 : : " b.hex as buildid, n.mtime, 'F' as sourcetype, f0.name as source0, n.mtime as mtime, null as source1\n"
254 : : " from " BUILDIDS "_buildids b, " BUILDIDS "_files_v f0, " BUILDIDS "_f_de n\n"
255 : : " where b.id = n.buildid and f0.id = n.file and n.debuginfo_p = 1\n"
256 : : "union all select\n"
257 : : " b.hex as buildid, n.mtime, 'R' as sourcetype, f0.name as source0, n.mtime as mtime, f1.name as source1\n"
258 : : " from " BUILDIDS "_buildids b, " BUILDIDS "_files_v f0, " BUILDIDS "_files_v f1, " BUILDIDS "_r_de n\n"
259 : : " where b.id = n.buildid and f0.id = n.file and f1.id = n.content and n.debuginfo_p = 1\n"
260 : : ";"
261 : : // ... and for E queries
262 : : "create view if not exists " BUILDIDS "_query_e as \n"
263 : : "select\n"
264 : : " b.hex as buildid, n.mtime, 'F' as sourcetype, f0.name as source0, n.mtime as mtime, null as source1\n"
265 : : " from " BUILDIDS "_buildids b, " BUILDIDS "_files_v f0, " BUILDIDS "_f_de n\n"
266 : : " where b.id = n.buildid and f0.id = n.file and n.executable_p = 1\n"
267 : : "union all select\n"
268 : : " b.hex as buildid, n.mtime, 'R' as sourcetype, f0.name as source0, n.mtime as mtime, f1.name as source1\n"
269 : : " from " BUILDIDS "_buildids b, " BUILDIDS "_files_v f0, " BUILDIDS "_files_v f1, " BUILDIDS "_r_de n\n"
270 : : " where b.id = n.buildid and f0.id = n.file and f1.id = n.content and n.executable_p = 1\n"
271 : : ";"
272 : : // ... and for S queries
273 : : "create view if not exists " BUILDIDS "_query_s as \n"
274 : : "select\n"
275 : : " b.hex as buildid, fs.name as artifactsrc, 'F' as sourcetype, f0.name as source0, n.mtime as mtime, null as source1, null as source0ref\n"
276 : : " from " BUILDIDS "_buildids b, " BUILDIDS "_files_v f0, " BUILDIDS "_files_v fs, " BUILDIDS "_f_s n\n"
277 : : " where b.id = n.buildid and f0.id = n.file and fs.id = n.artifactsrc\n"
278 : : "union all select\n"
279 : : " b.hex as buildid, f1.name as artifactsrc, 'R' as sourcetype, f0.name as source0, sd.mtime as mtime, f1.name as source1, fsref.name as source0ref\n"
280 : : " from " BUILDIDS "_buildids b, " BUILDIDS "_files_v f0, " BUILDIDS "_files_v f1, " BUILDIDS "_files_v fsref, "
281 : : " " BUILDIDS "_r_sdef sd, " BUILDIDS "_r_sref sr, " BUILDIDS "_r_de sde\n"
282 : : " where b.id = sr.buildid and f0.id = sd.file and fsref.id = sde.file and f1.id = sd.content\n"
283 : : " and sr.artifactsrc = sd.content and sde.buildid = sr.buildid\n"
284 : : ";"
285 : : // and for startup overview counts
286 : : "drop view if exists " BUILDIDS "_stats;\n"
287 : : "create view if not exists " BUILDIDS "_stats as\n"
288 : : " select 'file d/e' as label,count(*) as quantity from " BUILDIDS "_f_de\n"
289 : : "union all select 'file s',count(*) from " BUILDIDS "_f_s\n"
290 : : "union all select 'archive d/e',count(*) from " BUILDIDS "_r_de\n"
291 : : "union all select 'archive sref',count(*) from " BUILDIDS "_r_sref\n"
292 : : "union all select 'archive sdef',count(*) from " BUILDIDS "_r_sdef\n"
293 : : "union all select 'buildids',count(*) from " BUILDIDS "_buildids\n"
294 : : "union all select 'filenames',count(*) from " BUILDIDS "_files\n"
295 : : "union all select 'fileparts',count(*) from " BUILDIDS "_fileparts\n"
296 : : "union all select 'files scanned (#)',count(*) from " BUILDIDS "_file_mtime_scanned\n"
297 : : "union all select 'files scanned (mb)',coalesce(sum(size)/1024/1024,0) from " BUILDIDS "_file_mtime_scanned\n"
298 : : #if SQLITE_VERSION_NUMBER >= 3016000
299 : : "union all select 'index db size (mb)',page_count*page_size/1024/1024 as size FROM pragma_page_count(), pragma_page_size()\n"
300 : : #endif
301 : : ";\n"
302 : :
303 : : // schema change history & garbage collection
304 : : //
305 : : // XXX: we could have migration queries here to bring prior-schema
306 : : // data over instead of just dropping it. But that could incur
307 : : // doubled storage costs.
308 : : //
309 : : // buildids10: split the _files table into _parts
310 : : "" // <<< we are here
311 : : // buildids9: widen the mtime_scanned table
312 : : "DROP VIEW IF EXISTS buildids9_stats;\n"
313 : : "DROP INDEX IF EXISTS buildids9_r_de_idx;\n"
314 : : "DROP INDEX IF EXISTS buildids9_f_de_idx;\n"
315 : : "DROP VIEW IF EXISTS buildids9_query_s;\n"
316 : : "DROP VIEW IF EXISTS buildids9_query_e;\n"
317 : : "DROP VIEW IF EXISTS buildids9_query_d;\n"
318 : : "DROP TABLE IF EXISTS buildids9_r_sdef;\n"
319 : : "DROP TABLE IF EXISTS buildids9_r_sref;\n"
320 : : "DROP TABLE IF EXISTS buildids9_r_de;\n"
321 : : "DROP TABLE IF EXISTS buildids9_f_s;\n"
322 : : "DROP TABLE IF EXISTS buildids9_f_de;\n"
323 : : "DROP TABLE IF EXISTS buildids9_file_mtime_scanned;\n"
324 : : "DROP TABLE IF EXISTS buildids9_buildids;\n"
325 : : "DROP TABLE IF EXISTS buildids9_files;\n"
326 : : // buildids8: slim the sref table
327 : : "drop table if exists buildids8_f_de;\n"
328 : : "drop table if exists buildids8_f_s;\n"
329 : : "drop table if exists buildids8_r_de;\n"
330 : : "drop table if exists buildids8_r_sref;\n"
331 : : "drop table if exists buildids8_r_sdef;\n"
332 : : "drop table if exists buildids8_file_mtime_scanned;\n"
333 : : "drop table if exists buildids8_files;\n"
334 : : "drop table if exists buildids8_buildids;\n"
335 : : // buildids7: separate _norm table into dense subtype tables
336 : : "drop table if exists buildids7_f_de;\n"
337 : : "drop table if exists buildids7_f_s;\n"
338 : : "drop table if exists buildids7_r_de;\n"
339 : : "drop table if exists buildids7_r_sref;\n"
340 : : "drop table if exists buildids7_r_sdef;\n"
341 : : "drop table if exists buildids7_file_mtime_scanned;\n"
342 : : "drop table if exists buildids7_files;\n"
343 : : "drop table if exists buildids7_buildids;\n"
344 : : // buildids6: drop bolo/rfolo again, represent sources / rpmcontents in main table
345 : : "drop table if exists buildids6_norm;\n"
346 : : "drop table if exists buildids6_files;\n"
347 : : "drop table if exists buildids6_buildids;\n"
348 : : "drop view if exists buildids6;\n"
349 : : // buildids5: redefine srcfile1 column to be '.'-less (for rpms)
350 : : "drop table if exists buildids5_norm;\n"
351 : : "drop table if exists buildids5_files;\n"
352 : : "drop table if exists buildids5_buildids;\n"
353 : : "drop table if exists buildids5_bolo;\n"
354 : : "drop table if exists buildids5_rfolo;\n"
355 : : "drop view if exists buildids5;\n"
356 : : // buildids4: introduce rpmfile RFOLO
357 : : "drop table if exists buildids4_norm;\n"
358 : : "drop table if exists buildids4_files;\n"
359 : : "drop table if exists buildids4_buildids;\n"
360 : : "drop table if exists buildids4_bolo;\n"
361 : : "drop table if exists buildids4_rfolo;\n"
362 : : "drop view if exists buildids4;\n"
363 : : // buildids3*: split out srcfile BOLO
364 : : "drop table if exists buildids3_norm;\n"
365 : : "drop table if exists buildids3_files;\n"
366 : : "drop table if exists buildids3_buildids;\n"
367 : : "drop table if exists buildids3_bolo;\n"
368 : : "drop view if exists buildids3;\n"
369 : : // buildids2: normalized buildid and filenames into interning tables;
370 : : "drop table if exists buildids2_norm;\n"
371 : : "drop table if exists buildids2_files;\n"
372 : : "drop table if exists buildids2_buildids;\n"
373 : : "drop view if exists buildids2;\n"
374 : : // buildids1: made buildid and artifacttype NULLable, to represent cached-negative
375 : : // lookups from sources, e.g. files or rpms that contain no buildid-indexable content
376 : : "drop table if exists buildids1;\n"
377 : : // buildids: original
378 : : "drop table if exists buildids;\n"
379 : : ;
380 : :
381 : : static const char DEBUGINFOD_SQLITE_CLEANUP_DDL[] =
382 : : "pragma wal_checkpoint = truncate;\n" // clean out any preexisting wal file
383 : : ;
384 : :
385 : :
386 : :
387 : :
388 : : /* Name and version of program. */
389 : : ARGP_PROGRAM_VERSION_HOOK_DEF = print_version;
390 : :
391 : : /* Bug report address. */
392 : : ARGP_PROGRAM_BUG_ADDRESS_DEF = PACKAGE_BUGREPORT;
393 : :
394 : : /* Definitions of arguments for argp functions. */
395 : : static const struct argp_option options[] =
396 : : {
397 : : { NULL, 0, NULL, 0, "Scanners:", 1 },
398 : : { "scan-file-dir", 'F', NULL, 0, "Enable ELF/DWARF file scanning.", 0 },
399 : : { "scan-rpm-dir", 'R', NULL, 0, "Enable RPM scanning.", 0 },
400 : : { "scan-deb-dir", 'U', NULL, 0, "Enable DEB scanning.", 0 },
401 : : { "scan-archive", 'Z', "EXT=CMD", 0, "Enable arbitrary archive scanning.", 0 },
402 : : // "source-oci-imageregistry" ...
403 : :
404 : : { NULL, 0, NULL, 0, "Options:", 2 },
405 : : { "logical", 'L', NULL, 0, "Follow symlinks, default=ignore.", 0 },
406 : : { "rescan-time", 't', "SECONDS", 0, "Number of seconds to wait between rescans, 0=disable.", 0 },
407 : : { "groom-time", 'g', "SECONDS", 0, "Number of seconds to wait between database grooming, 0=disable.", 0 },
408 : : { "maxigroom", 'G', NULL, 0, "Run a complete database groom/shrink pass at startup.", 0 },
409 : : { "concurrency", 'c', "NUM", 0, "Limit scanning thread concurrency to NUM, default=#CPUs.", 0 },
410 : : { "connection-pool", 'C', "NUM", OPTION_ARG_OPTIONAL,
411 : : "Use webapi connection pool with NUM threads, default=unlim.", 0 },
412 : : { "include", 'I', "REGEX", 0, "Include files matching REGEX, default=all.", 0 },
413 : : { "exclude", 'X', "REGEX", 0, "Exclude files matching REGEX, default=none.", 0 },
414 : : { "port", 'p', "NUM", 0, "HTTP port to listen on, default 8002.", 0 },
415 : : { "database", 'd', "FILE", 0, "Path to sqlite database.", 0 },
416 : : { "ddl", 'D', "SQL", 0, "Apply extra sqlite ddl/pragma to connection.", 0 },
417 : : { "verbose", 'v', NULL, 0, "Increase verbosity.", 0 },
418 : : { "regex-groom", 'r', NULL, 0,"Uses regexes from -I and -X arguments to groom the database.",0},
419 : : #define ARGP_KEY_FDCACHE_FDS 0x1001
420 : : { "fdcache-fds", ARGP_KEY_FDCACHE_FDS, "NUM", 0, "Maximum number of archive files to keep in fdcache.", 0 },
421 : : #define ARGP_KEY_FDCACHE_MBS 0x1002
422 : : { "fdcache-mbs", ARGP_KEY_FDCACHE_MBS, "MB", 0, "Maximum total size of archive file fdcache.", 0 },
423 : : #define ARGP_KEY_FDCACHE_PREFETCH 0x1003
424 : : { "fdcache-prefetch", ARGP_KEY_FDCACHE_PREFETCH, "NUM", 0, "Number of archive files to prefetch into fdcache.", 0 },
425 : : #define ARGP_KEY_FDCACHE_MINTMP 0x1004
426 : : { "fdcache-mintmp", ARGP_KEY_FDCACHE_MINTMP, "NUM", 0, "Minimum free space% on tmpdir.", 0 },
427 : : #define ARGP_KEY_FDCACHE_PREFETCH_MBS 0x1005
428 : : { "fdcache-prefetch-mbs", ARGP_KEY_FDCACHE_PREFETCH_MBS, "MB", 0,"Megabytes allocated to the \
429 : : prefetch cache.", 0},
430 : : #define ARGP_KEY_FDCACHE_PREFETCH_FDS 0x1006
431 : : { "fdcache-prefetch-fds", ARGP_KEY_FDCACHE_PREFETCH_FDS, "NUM", 0,"Number of files allocated to the \
432 : : prefetch cache.", 0},
433 : : #define ARGP_KEY_FORWARDED_TTL_LIMIT 0x1007
434 : : {"forwarded-ttl-limit", ARGP_KEY_FORWARDED_TTL_LIMIT, "NUM", 0, "Limit of X-Forwarded-For hops, default 8.", 0},
435 : : #define ARGP_KEY_PASSIVE 0x1008
436 : : { "passive", ARGP_KEY_PASSIVE, NULL, 0, "Do not scan or groom, read-only database.", 0 },
437 : : #define ARGP_KEY_DISABLE_SOURCE_SCAN 0x1009
438 : : { "disable-source-scan", ARGP_KEY_DISABLE_SOURCE_SCAN, NULL, 0, "Do not scan dwarf source info.", 0 },
439 : : #define ARGP_SCAN_CHECKPOINT 0x100A
440 : : { "scan-checkpoint", ARGP_SCAN_CHECKPOINT, "NUM", 0, "Number of files scanned before a WAL checkpoint.", 0 },
441 : : { NULL, 0, NULL, 0, NULL, 0 },
442 : : };
443 : :
444 : : /* Short description of program. */
445 : : static const char doc[] = "Serve debuginfo-related content across HTTP from files under PATHs.";
446 : :
447 : : /* Strings for arguments in help texts. */
448 : : static const char args_doc[] = "[PATH ...]";
449 : :
450 : : /* Prototype for option handler. */
451 : : static error_t parse_opt (int key, char *arg, struct argp_state *state);
452 : :
453 : : static unsigned default_concurrency();
454 : :
455 : : /* Data structure to communicate with argp functions. */
456 : : static struct argp argp =
457 : : {
458 : : options, parse_opt, args_doc, doc, NULL, NULL, NULL
459 : : };
460 : :
461 : :
462 : : static string db_path;
463 : : static sqlite3 *db; // single connection, serialized across all our threads!
464 : : static sqlite3 *dbq; // webapi query-servicing readonly connection, serialized ditto!
465 : : static unsigned verbose;
466 : : static volatile sig_atomic_t interrupted = 0;
467 : : static volatile sig_atomic_t forced_rescan_count = 0;
468 : : static volatile sig_atomic_t sigusr1 = 0;
469 : : static volatile sig_atomic_t forced_groom_count = 0;
470 : : static volatile sig_atomic_t sigusr2 = 0;
471 : : static unsigned http_port = 8002;
472 : : static unsigned rescan_s = 300;
473 : : static unsigned groom_s = 86400;
474 : : static bool maxigroom = false;
475 : : static unsigned concurrency = default_concurrency();
476 : : static int connection_pool = 0;
477 : : static set<string> source_paths;
478 : : static bool scan_files = false;
479 : : static map<string,string> scan_archives;
480 : : static vector<string> extra_ddl;
481 : : static regex_t file_include_regex;
482 : : static regex_t file_exclude_regex;
483 : : static bool regex_groom = false;
484 : : static bool traverse_logical;
485 : : static long fdcache_fds;
486 : : static long fdcache_mbs;
487 : : static long fdcache_prefetch;
488 : : static long fdcache_mintmp;
489 : : static long fdcache_prefetch_mbs;
490 : : static long fdcache_prefetch_fds;
491 : : static unsigned forwarded_ttl_limit = 8;
492 : : static bool scan_source_info = true;
493 : : static string tmpdir;
494 : : static bool passive_p = false;
495 : : static long scan_checkpoint = 256;
496 : :
497 : : static void set_metric(const string& key, double value);
498 : : // static void inc_metric(const string& key);
499 : : static void set_metric(const string& metric,
500 : : const string& lname, const string& lvalue,
501 : : double value);
502 : : static void inc_metric(const string& metric,
503 : : const string& lname, const string& lvalue);
504 : : static void add_metric(const string& metric,
505 : : const string& lname, const string& lvalue,
506 : : double value);
507 : : static void inc_metric(const string& metric,
508 : : const string& lname, const string& lvalue,
509 : : const string& rname, const string& rvalue);
510 : : static void add_metric(const string& metric,
511 : : const string& lname, const string& lvalue,
512 : : const string& rname, const string& rvalue,
513 : : double value);
514 : :
515 : :
516 : : class tmp_inc_metric { // a RAII style wrapper for exception-safe scoped increment & decrement
517 : : string m, n, v;
518 : : public:
519 : 2070 : tmp_inc_metric(const string& mname, const string& lname, const string& lvalue):
520 [ + - + - ]: 2070 : m(mname), n(lname), v(lvalue)
521 : : {
522 [ + - ]: 2070 : add_metric (m, n, v, 1);
523 [ - - - - : 2070 : }
- - ]
524 : 2070 : ~tmp_inc_metric()
525 : : {
526 : 2070 : add_metric (m, n, v, -1);
527 [ - + - + : 2070 : }
- + ]
528 : : };
529 : :
530 : : class tmp_ms_metric { // a RAII style wrapper for exception-safe scoped timing
531 : : string m, n, v;
532 : : struct timespec ts_start;
533 : : public:
534 : 98073 : tmp_ms_metric(const string& mname, const string& lname, const string& lvalue):
535 [ + - + - ]: 98073 : m(mname), n(lname), v(lvalue)
536 : : {
537 : 98064 : clock_gettime (CLOCK_MONOTONIC, & ts_start);
538 [ - - - - ]: 98095 : }
539 : 98109 : ~tmp_ms_metric()
540 : : {
541 : 98109 : struct timespec ts_end;
542 : 98109 : clock_gettime (CLOCK_MONOTONIC, & ts_end);
543 : 98114 : double deltas = (ts_end.tv_sec - ts_start.tv_sec)
544 : 98114 : + (ts_end.tv_nsec - ts_start.tv_nsec)/1.e9;
545 : :
546 [ + - ]: 98114 : add_metric (m + "_milliseconds_sum", n, v, (deltas*1000.0));
547 [ + + ]: 98123 : inc_metric (m + "_milliseconds_count", n, v);
548 [ + + - + : 172863 : }
- + ]
549 : : };
550 : :
551 : :
552 : : /* Handle program arguments. */
553 : : static error_t
554 : 1124 : parse_opt (int key, char *arg,
555 : : struct argp_state *state __attribute__ ((unused)))
556 : : {
557 : 1124 : int rc;
558 [ + + + + : 1124 : switch (key)
+ + + + -
+ + - - +
+ + + + +
+ + + + +
+ + - +
+ ]
559 : : {
560 : 278 : case 'v': verbose ++; break;
561 : 72 : case 'd':
562 : : /* When using the in-memory database make sure it is shareable,
563 : : so we can open it twice as read/write and read-only. */
564 [ + + ]: 72 : if (strcmp (arg, ":memory:") == 0)
565 : 1138 : db_path = "file::memory:?cache=shared";
566 : : else
567 [ + - ]: 116 : db_path = string(arg);
568 : : break;
569 : 72 : case 'p': http_port = (unsigned) atoi(arg);
570 [ + - ]: 72 : if (http_port == 0 || http_port > 65535)
571 : 0 : argp_failure(state, 1, EINVAL, "port number");
572 : : break;
573 : 48 : case 'F': scan_files = true; break;
574 : 22 : case 'R':
575 [ + - + - : 22 : scan_archives[".rpm"]="cat"; // libarchive groks rpm natively
- + ]
576 : 22 : break;
577 : 14 : case 'U':
578 [ + - + - : 14 : scan_archives[".deb"]="(bsdtar -O -x -f - data.tar\\*)<";
- + ]
579 [ + - + - : 14 : scan_archives[".ddeb"]="(bsdtar -O -x -f - data.tar\\*)<";
- + ]
580 [ + - + - : 14 : scan_archives[".ipk"]="(bsdtar -O -x -f - data.tar\\*)<";
- + ]
581 : : // .udeb too?
582 : 14 : break;
583 : 38 : case 'Z':
584 : 38 : {
585 [ - + ]: 38 : char* extension = strchr(arg, '=');
586 [ - + ]: 38 : if (arg[0] == '\0')
587 : 0 : argp_failure(state, 1, EINVAL, "missing EXT");
588 [ + + ]: 38 : else if (extension)
589 [ + - + - : 20 : scan_archives[string(arg, (extension-arg))]=string(extension+1);
- + - + -
- ]
590 : : else
591 [ + - + - : 18 : scan_archives[string(arg)]=string("cat");
- + - + -
- ]
592 : : }
593 : : break;
594 : 8 : case 'L':
595 [ - + ]: 8 : if (passive_p)
596 : 0 : argp_failure(state, 1, EINVAL, "-L option inconsistent with passive mode");
597 : 8 : traverse_logical = true;
598 : 8 : break;
599 : 0 : case 'D':
600 [ # # ]: 0 : if (passive_p)
601 : 0 : argp_failure(state, 1, EINVAL, "-D option inconsistent with passive mode");
602 [ # # ]: 0 : extra_ddl.push_back(string(arg));
603 : 0 : break;
604 : 58 : case 't':
605 [ - + ]: 58 : if (passive_p)
606 : 0 : argp_failure(state, 1, EINVAL, "-t option inconsistent with passive mode");
607 : 58 : rescan_s = (unsigned) atoi(arg);
608 : 58 : break;
609 : 58 : case 'g':
610 [ - + ]: 58 : if (passive_p)
611 : 0 : argp_failure(state, 1, EINVAL, "-g option inconsistent with passive mode");
612 : 58 : groom_s = (unsigned) atoi(arg);
613 : 58 : break;
614 : 0 : case 'G':
615 [ # # ]: 0 : if (passive_p)
616 : 0 : argp_failure(state, 1, EINVAL, "-G option inconsistent with passive mode");
617 : 0 : maxigroom = true;
618 : 0 : break;
619 : 0 : case 'c':
620 [ # # ]: 0 : if (passive_p)
621 : 0 : argp_failure(state, 1, EINVAL, "-c option inconsistent with passive mode");
622 : 0 : concurrency = (unsigned) atoi(arg);
623 [ # # ]: 0 : if (concurrency < 1) concurrency = 1;
624 : : break;
625 : 6 : case 'C':
626 [ + + ]: 6 : if (arg)
627 : : {
628 : 4 : connection_pool = atoi(arg);
629 [ + - ]: 4 : if (connection_pool < 2)
630 : 0 : argp_failure(state, 1, EINVAL, "-C NUM minimum 2");
631 : : }
632 : : break;
633 : 4 : case 'I':
634 : : // NB: no problem with unconditional free here - an earlier failed regcomp would exit program
635 [ - + ]: 4 : if (passive_p)
636 : 0 : argp_failure(state, 1, EINVAL, "-I option inconsistent with passive mode");
637 : 4 : regfree (&file_include_regex);
638 : 4 : rc = regcomp (&file_include_regex, arg, REG_EXTENDED|REG_NOSUB);
639 [ + - ]: 4 : if (rc != 0)
640 : 0 : argp_failure(state, 1, EINVAL, "regular expression");
641 : : break;
642 : 6 : case 'X':
643 [ - + ]: 6 : if (passive_p)
644 : 0 : argp_failure(state, 1, EINVAL, "-X option inconsistent with passive mode");
645 : 6 : regfree (&file_exclude_regex);
646 : 6 : rc = regcomp (&file_exclude_regex, arg, REG_EXTENDED|REG_NOSUB);
647 [ + - ]: 6 : if (rc != 0)
648 : 0 : argp_failure(state, 1, EINVAL, "regular expression");
649 : : break;
650 : 4 : case 'r':
651 [ - + ]: 4 : if (passive_p)
652 : 0 : argp_failure(state, 1, EINVAL, "-r option inconsistent with passive mode");
653 : 4 : regex_groom = true;
654 : 4 : break;
655 : 10 : case ARGP_KEY_FDCACHE_FDS:
656 : 10 : fdcache_fds = atol (arg);
657 : 10 : break;
658 : 4 : case ARGP_KEY_FDCACHE_MBS:
659 : 4 : fdcache_mbs = atol (arg);
660 : 4 : break;
661 : 2 : case ARGP_KEY_FDCACHE_PREFETCH:
662 : 2 : fdcache_prefetch = atol (arg);
663 : 2 : break;
664 : 2 : case ARGP_KEY_FDCACHE_MINTMP:
665 : 2 : fdcache_mintmp = atol (arg);
666 [ + - ]: 2 : if( fdcache_mintmp > 100 || fdcache_mintmp < 0 )
667 : 0 : argp_failure(state, 1, EINVAL, "fdcache mintmp percent");
668 : : break;
669 : 4 : case ARGP_KEY_FORWARDED_TTL_LIMIT:
670 : 4 : forwarded_ttl_limit = (unsigned) atoi(arg);
671 : 4 : break;
672 : 100 : case ARGP_KEY_ARG:
673 [ + - ]: 100 : source_paths.insert(string(arg));
674 : 100 : break;
675 : 10 : case ARGP_KEY_FDCACHE_PREFETCH_FDS:
676 : 10 : fdcache_prefetch_fds = atol(arg);
677 [ + - ]: 10 : if ( fdcache_prefetch_fds < 0)
678 : 0 : argp_failure(state, 1, EINVAL, "fdcache prefetch fds");
679 : : break;
680 : 2 : case ARGP_KEY_FDCACHE_PREFETCH_MBS:
681 : 2 : fdcache_prefetch_mbs = atol(arg);
682 [ + - ]: 2 : if ( fdcache_prefetch_mbs < 0)
683 : 0 : argp_failure(state, 1, EINVAL, "fdcache prefetch mbs");
684 : : break;
685 : 2 : case ARGP_KEY_PASSIVE:
686 : 2 : passive_p = true;
687 [ + - ]: 2 : if (source_paths.size() > 0
688 [ + - ]: 2 : || maxigroom
689 [ + - ]: 2 : || extra_ddl.size() > 0
690 [ + - + - ]: 4 : || traverse_logical)
691 : : // other conflicting options tricky to check
692 : 0 : argp_failure(state, 1, EINVAL, "inconsistent options with passive mode");
693 : : break;
694 : 0 : case ARGP_KEY_DISABLE_SOURCE_SCAN:
695 : 0 : scan_source_info = false;
696 : 0 : break;
697 : 2 : case ARGP_SCAN_CHECKPOINT:
698 : 2 : scan_checkpoint = atol (arg);
699 [ + - ]: 2 : if (scan_checkpoint < 0)
700 : 0 : argp_failure(state, 1, EINVAL, "scan checkpoint");
701 : : break;
702 : : // case 'h': argp_state_help (state, stderr, ARGP_HELP_LONG|ARGP_HELP_EXIT_OK);
703 : : default: return ARGP_ERR_UNKNOWN;
704 : : }
705 : :
706 : : return 0;
707 : : }
708 : :
709 : :
710 : : ////////////////////////////////////////////////////////////////////////
711 : :
712 : :
713 : : static void add_mhd_response_header (struct MHD_Response *r,
714 : : const char *h, const char *v);
715 : :
716 : : // represent errors that may get reported to an ostream and/or a libmicrohttpd connection
717 : :
718 : 10 : struct reportable_exception
719 : : {
720 : : int code;
721 : : string message;
722 : :
723 [ - - + - : 86 : reportable_exception(int c, const string& m): code(c), message(m) {}
- - + - +
- ]
724 [ - - - - : 601 : reportable_exception(const string& m): code(503), message(m) {}
- - - - +
- - - - -
+ - - - -
- - - + -
- - ]
725 : : reportable_exception(): code(503), message() {}
726 : :
727 : : void report(ostream& o) const; // defined under obatched() class below
728 : :
729 : 630 : MHD_RESULT mhd_send_response(MHD_Connection* c) const {
730 : 1260 : MHD_Response* r = MHD_create_response_from_buffer (message.size(),
731 : 630 : (void*) message.c_str(),
732 : : MHD_RESPMEM_MUST_COPY);
733 : 630 : add_mhd_response_header (r, "Content-Type", "text/plain");
734 : 630 : MHD_RESULT rc = MHD_queue_response (c, code, r);
735 : 630 : MHD_destroy_response (r);
736 : 630 : return rc;
737 : : }
738 : : };
739 : :
740 : :
741 : : struct sqlite_exception: public reportable_exception
742 : : {
743 : 0 : sqlite_exception(int rc, const string& msg):
744 [ # # # # : 0 : reportable_exception(string("sqlite3 error: ") + msg + ": " + string(sqlite3_errstr(rc) ?: "?")) {
# # # # #
# # # # #
# # # # #
# ]
745 [ # # # # : 0 : inc_metric("error_count","sqlite3",sqlite3_errstr(rc));
# # # # #
# # # # #
# # # # ]
746 [ # # ]: 0 : }
747 : : };
748 : :
749 [ + - - - ]: 5 : struct libc_exception: public reportable_exception
750 : : {
751 : 595 : libc_exception(int rc, const string& msg):
752 [ - + + - : 2380 : reportable_exception(string("libc error: ") + msg + ": " + string(strerror(rc) ?: "?")) {
+ - + - +
- - + - +
- + + - -
- ]
753 [ + - + - : 1190 : inc_metric("error_count","libc",strerror(rc));
+ - + - -
+ + - - -
- - ]
754 [ - - ]: 595 : }
755 : : };
756 : :
757 : :
758 : : struct archive_exception: public reportable_exception
759 : : {
760 : 0 : archive_exception(const string& msg):
761 [ # # # # : 0 : reportable_exception(string("libarchive error: ") + msg) {
# # ]
762 [ # # # # : 0 : inc_metric("error_count","libarchive",msg);
# # # # #
# ]
763 [ # # ]: 0 : }
764 : 0 : archive_exception(struct archive* a, const string& msg):
765 [ # # # # : 0 : reportable_exception(string("libarchive error: ") + msg + ": " + string(archive_error_string(a) ?: "?")) {
# # # # #
# # # # #
# # # # #
# ]
766 [ # # # # : 0 : inc_metric("error_count","libarchive",msg + ": " + string(archive_error_string(a) ?: "?"));
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
767 [ # # ]: 0 : }
768 : : };
769 : :
770 : :
771 : : struct elfutils_exception: public reportable_exception
772 : : {
773 : 0 : elfutils_exception(int rc, const string& msg):
774 [ # # # # : 0 : reportable_exception(string("elfutils error: ") + msg + ": " + string(elf_errmsg(rc) ?: "?")) {
# # # # #
# # # # #
# # # # #
# ]
775 [ # # # # : 0 : inc_metric("error_count","elfutils",elf_errmsg(rc));
# # # # #
# # # # #
# # # # ]
776 [ # # ]: 0 : }
777 : : };
778 : :
779 : :
780 : : ////////////////////////////////////////////////////////////////////////
781 : :
782 : : template <typename Payload>
783 : : class workq
784 : : {
785 : : unordered_set<Payload> q; // eliminate duplicates
786 : : mutex mtx;
787 : : condition_variable cv;
788 : : bool dead;
789 : : unsigned idlers; // number of threads busy with wait_idle / done_idle
790 : : unsigned fronters; // number of threads busy with wait_front / done_front
791 : :
792 : : public:
793 : 72 : workq() { dead = false; idlers = 0; fronters = 0; }
794 : 72 : ~workq() {}
795 : :
796 : 940 : void push_back(const Payload& p)
797 : : {
798 : 940 : unique_lock<mutex> lock(mtx);
799 [ + - ]: 940 : q.insert (p);
800 [ + - + - : 1880 : set_metric("thread_work_pending","role","scan", q.size());
+ - + - -
+ - + - -
- - ]
801 : 940 : cv.notify_all();
802 : 940 : }
803 : :
804 : : // kill this workqueue, wake up all idlers / scanners
805 : 72 : void nuke() {
806 : 72 : unique_lock<mutex> lock(mtx);
807 : : // optional: q.clear();
808 : 72 : dead = true;
809 : 72 : cv.notify_all();
810 : 72 : }
811 : :
812 : : // clear the workqueue, when scanning is interrupted with USR2
813 : 0 : void clear() {
814 : 0 : unique_lock<mutex> lock(mtx);
815 : 0 : q.clear();
816 [ # # # # : 0 : set_metric("thread_work_pending","role","scan", q.size());
# # # # #
# # # # #
# # ]
817 : : // NB: there may still be some live fronters
818 : 0 : cv.notify_all(); // maybe wake up waiting idlers
819 : 0 : }
820 : :
821 : : // block this scanner thread until there is work to do and no active idler
822 : 1196 : bool wait_front (Payload& p)
823 : : {
824 : 1196 : unique_lock<mutex> lock(mtx);
825 [ + + + + : 4789 : while (!dead && (q.size() == 0 || idlers > 0))
+ + ]
826 [ + - ]: 3593 : cv.wait(lock);
827 [ + + ]: 1196 : if (dead)
828 : : return false;
829 : : else
830 : : {
831 [ + - ]: 940 : p = * q.begin();
832 : 940 : q.erase (q.begin());
833 : 940 : fronters ++; // prevent idlers from starting awhile, even if empty q
834 [ + - + - : 1880 : set_metric("thread_work_pending","role","scan", q.size());
+ - + - -
+ - + - -
- - - - ]
835 : : // NB: don't wake up idlers yet! The consumer is busy
836 : : // processing this element until it calls done_front().
837 : 940 : return true;
838 : : }
839 : 1196 : }
840 : :
841 : : // notify waitq that scanner thread is done with that last item
842 : 940 : void done_front ()
843 : : {
844 : 940 : unique_lock<mutex> lock(mtx);
845 : 940 : fronters --;
846 [ + + + + ]: 940 : if (q.size() == 0 && fronters == 0)
847 : 95 : cv.notify_all(); // maybe wake up waiting idlers
848 : 940 : }
849 : :
850 : : // block this idler thread until there is no work to do
851 : 613 : void wait_idle ()
852 : : {
853 : 613 : unique_lock<mutex> lock(mtx);
854 : 613 : cv.notify_all(); // maybe wake up waiting scanners
855 [ + + + + : 666 : while (!dead && ((q.size() != 0) || fronters > 0))
+ + ]
856 [ + - ]: 53 : cv.wait(lock);
857 [ + - ]: 613 : idlers ++;
858 : 613 : }
859 : :
860 : 543 : void done_idle ()
861 : : {
862 : 543 : unique_lock<mutex> lock(mtx);
863 : 543 : idlers --;
864 : 543 : cv.notify_all(); // maybe wake up waiting scanners, but probably not (shutting down)
865 : 543 : }
866 : : };
867 : :
868 : : typedef struct stat stat_t;
869 : : typedef pair<string,stat_t> scan_payload;
870 : : inline bool operator< (const scan_payload& a, const scan_payload& b)
871 : : {
872 : : return a.first < b.first; // don't bother compare the stat fields
873 : : }
874 : :
875 : : namespace std { // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=56480
876 : : template<> struct hash<::scan_payload>
877 : : {
878 : 3576 : std::size_t operator() (const ::scan_payload& p) const noexcept
879 : : {
880 [ + + + + ]: 3576 : return hash<string>()(p.first);
881 : : }
882 : : };
883 : : template<> struct equal_to<::scan_payload>
884 : : {
885 : 294 : std::size_t operator() (const ::scan_payload& a, const ::scan_payload& b) const noexcept
886 : : {
887 [ - + - - ]: 294 : return a.first == b.first;
888 : : }
889 : : };
890 : : }
891 : :
892 : : static workq<scan_payload> scanq; // just a single one
893 : : // producer & idler: thread_main_fts_source_paths()
894 : : // consumer: thread_main_scanner()
895 : : // idler: thread_main_groom()
896 : :
897 : :
898 : : ////////////////////////////////////////////////////////////////////////
899 : :
900 : : // Unique set is a thread-safe structure that lends 'ownership' of a value
901 : : // to a thread. Other threads requesting the same thing are made to wait.
902 : : // It's like a semaphore-on-demand.
903 : : template <typename T>
904 : : class unique_set
905 : : {
906 : : private:
907 : : set<T> values;
908 : : mutex mtx;
909 : : condition_variable cv;
910 : : public:
911 : 56 : unique_set() {}
912 : 56 : ~unique_set() {}
913 : :
914 : 1384 : void acquire(const T& value)
915 : : {
916 : 1384 : unique_lock<mutex> lock(mtx);
917 [ + + ]: 5592 : while (values.find(value) != values.end())
918 [ + - ]: 4208 : cv.wait(lock);
919 [ + - ]: 1384 : values.insert(value);
920 : 1384 : }
921 : :
922 : 1384 : void release(const T& value)
923 : : {
924 : 1384 : unique_lock<mutex> lock(mtx);
925 : : // assert (values.find(value) != values.end());
926 : 1384 : values.erase(value);
927 : 1384 : cv.notify_all();
928 : 1384 : }
929 : : };
930 : :
931 : :
932 : : // This is the object that's instantiate to uniquely hold a value in a
933 : : // RAII-pattern way.
934 : : template <typename T>
935 : : class unique_set_reserver
936 : : {
937 : : private:
938 : : unique_set<T>& please_hold;
939 : : T mine;
940 : : public:
941 : 1384 : unique_set_reserver(unique_set<T>& t, const T& value):
942 [ + - - - ]: 1384 : please_hold(t), mine(value) { please_hold.acquire(mine); }
943 [ + - ]: 1384 : ~unique_set_reserver() { please_hold.release(mine); }
944 : : };
945 : :
946 : :
947 : : ////////////////////////////////////////////////////////////////////////
948 : :
949 : : // periodic_barrier is a concurrency control object that lets N threads
950 : : // periodically (based on counter value) agree to wait at a barrier,
951 : : // let one of them carry out some work, then be set free
952 : :
953 : : class periodic_barrier
954 : : {
955 : : private:
956 : : unsigned period; // number of count() reports to trigger barrier activation
957 : : unsigned threads; // number of threads participating
958 : : mutex mtx; // protects all the following fields
959 : : unsigned counter; // count of count() reports in the current generation
960 : : unsigned generation; // barrier activation generation
961 : : unsigned waiting; // number of threads waiting for barrier
962 : : bool dead; // bring out your
963 : : condition_variable cv;
964 : : public:
965 : 64 : periodic_barrier(unsigned t, unsigned p):
966 : 64 : period(p), threads(t), counter(0), generation(0), waiting(0), dead(false) { }
967 : : virtual ~periodic_barrier() {}
968 : :
969 : : virtual void periodic_barrier_work() noexcept = 0;
970 : 64 : void nuke() {
971 : 64 : unique_lock<mutex> lock(mtx);
972 : 64 : dead = true;
973 : 64 : cv.notify_all();
974 : 64 : }
975 : :
976 : 1196 : void count()
977 : : {
978 : 1196 : unique_lock<mutex> lock(mtx);
979 : 1196 : unsigned prev_generation = this->generation;
980 [ + + ]: 1196 : if (counter < period-1) // normal case: counter just freely running
981 : : {
982 : 1080 : counter ++;
983 : 1080 : return;
984 : : }
985 [ + + ]: 116 : else if (counter == period-1) // we're the doer
986 : : {
987 : 30 : counter = period; // entering barrier holding phase
988 : 30 : cv.notify_all();
989 [ + + + + ]: 143 : while (waiting < threads-1 && !dead)
990 [ + - ]: 83 : cv.wait(lock);
991 : : // all other threads are now stuck in the barrier
992 : 30 : this->periodic_barrier_work(); // NB: we're holding the mutex the whole time
993 : : // reset for next barrier, releasing other waiters
994 : 30 : counter = 0;
995 : 30 : generation ++;
996 : 30 : cv.notify_all();
997 : 30 : return;
998 : : }
999 [ + - ]: 86 : else if (counter == period) // we're a waiter, in holding phase
1000 : : {
1001 : 86 : waiting ++;
1002 : 86 : cv.notify_all();
1003 [ + + + - : 308 : while (counter == period && generation == prev_generation && !dead)
+ + ]
1004 [ + - ]: 136 : cv.wait(lock);
1005 : 86 : waiting --;
1006 : 86 : return;
1007 : : }
1008 : 1196 : }
1009 : : };
1010 : :
1011 : :
1012 : :
1013 : : ////////////////////////////////////////////////////////////////////////
1014 : :
1015 : :
1016 : : // Print a standard timestamp.
1017 : : static ostream&
1018 : 22045 : timestamp (ostream &o)
1019 : : {
1020 : 22045 : char datebuf[80];
1021 : 22045 : char *now2 = NULL;
1022 : 22045 : time_t now_t = time(NULL);
1023 : 22046 : struct tm now;
1024 : 22046 : struct tm *nowp = gmtime_r (&now_t, &now);
1025 [ + - ]: 22044 : if (nowp)
1026 : : {
1027 : 22044 : (void) strftime (datebuf, sizeof (datebuf), "%c", nowp);
1028 : 22044 : now2 = datebuf;
1029 : : }
1030 : :
1031 : 22044 : return o << "[" << (now2 ? now2 : "") << "] "
1032 [ - + ]: 22044 : << "(" << getpid () << "/" << tid() << "): ";
1033 : : }
1034 : :
1035 : :
1036 : : // A little class that impersonates an ostream to the extent that it can
1037 : : // take << streaming operations. It batches up the bits into an internal
1038 : : // stringstream until it is destroyed; then flushes to the original ostream.
1039 : : // It adds a timestamp
1040 : : class obatched
1041 : : {
1042 : : private:
1043 : : ostream& o;
1044 : : stringstream stro;
1045 : : static mutex lock;
1046 : : public:
1047 : 22046 : obatched(ostream& oo, bool timestamp_p = true): o(oo)
1048 : : {
1049 [ + - ]: 22045 : if (timestamp_p)
1050 [ + - ]: 22045 : timestamp(stro);
1051 : 22046 : }
1052 : 22041 : ~obatched()
1053 : : {
1054 : 22041 : unique_lock<mutex> do_not_cross_the_streams(obatched::lock);
1055 [ + - ]: 22047 : o << stro.str();
1056 : 22047 : o.flush();
1057 : 22046 : }
1058 : : operator ostream& () { return stro; }
1059 [ - - + - : 16617 : template <typename T> ostream& operator << (const T& t) { stro << t; return stro; }
+ - + - +
- + - + -
- - - - -
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- - - + -
+ - + - +
- + - + -
+ - + - -
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
- - - - -
- + - - -
+ - - - -
- - - + -
- - + - -
- + - - -
- - + - +
- + - + -
- - + - -
- - - ]
1060 : : };
1061 : : mutex obatched::lock; // just the one, since cout/cerr iostreams are not thread-safe
1062 : :
1063 : :
1064 : 693 : void reportable_exception::report(ostream& o) const {
1065 [ + - + - ]: 693 : obatched(o) << message << endl;
1066 : 693 : }
1067 : :
1068 : :
1069 : : ////////////////////////////////////////////////////////////////////////
1070 : :
1071 : :
1072 : : // RAII style sqlite prepared-statement holder that matches { } block lifetime
1073 : :
1074 : : struct sqlite_ps
1075 : : {
1076 : : private:
1077 : : sqlite3* db;
1078 : : const string nickname;
1079 : : const string sql;
1080 : : sqlite3_stmt *pp;
1081 : :
1082 : : sqlite_ps(const sqlite_ps&); // make uncopyable
1083 : : sqlite_ps& operator=(const sqlite_ps &); // make unassignable
1084 : :
1085 : : public:
1086 [ + - - - ]: 6408 : sqlite_ps (sqlite3* d, const string& n, const string& s): db(d), nickname(n), sql(s) {
1087 : : // tmp_ms_metric tick("sqlite3","prep",nickname);
1088 [ + + ]: 6406 : if (verbose > 4)
1089 [ + - + - : 348 : obatched(clog) << nickname << " prep " << sql << endl;
+ - + - +
- - - ]
1090 [ + - ]: 6406 : int rc = sqlite3_prepare_v2 (db, sql.c_str(), -1 /* to \0 */, & this->pp, NULL);
1091 [ - + ]: 6409 : if (rc != SQLITE_OK)
1092 [ # # # # ]: 0 : throw sqlite_exception(rc, "prepare " + sql);
1093 : 6409 : }
1094 : :
1095 : 53920 : sqlite_ps& reset()
1096 : : {
1097 [ + - + - : 107839 : tmp_ms_metric tick("sqlite3","reset",nickname);
- + - - ]
1098 [ + - ]: 53919 : sqlite3_reset(this->pp);
1099 : 53924 : return *this;
1100 : 53916 : }
1101 : :
1102 : 57647 : sqlite_ps& bind(int parameter, const string& str)
1103 : : {
1104 [ + + ]: 57647 : if (verbose > 4)
1105 [ + - + - : 1418 : obatched(clog) << nickname << " bind " << parameter << "=" << str << endl;
+ - + - +
- + - ]
1106 : 57647 : int rc = sqlite3_bind_text (this->pp, parameter, str.c_str(), -1, SQLITE_TRANSIENT);
1107 [ - + ]: 57648 : if (rc != SQLITE_OK)
1108 [ # # # # ]: 0 : throw sqlite_exception(rc, "sqlite3 bind");
1109 : 57648 : return *this;
1110 : : }
1111 : :
1112 : 17702 : sqlite_ps& bind(int parameter, int64_t value)
1113 : : {
1114 [ + + ]: 17702 : if (verbose > 4)
1115 [ + - + - : 636 : obatched(clog) << nickname << " bind " << parameter << "=" << value << endl;
+ - + - +
- + - ]
1116 : 17702 : int rc = sqlite3_bind_int64 (this->pp, parameter, value);
1117 [ - + ]: 17704 : if (rc != SQLITE_OK)
1118 [ # # # # ]: 0 : throw sqlite_exception(rc, "sqlite3 bind");
1119 : 17704 : return *this;
1120 : : }
1121 : :
1122 : : sqlite_ps& bind(int parameter)
1123 : : {
1124 : : if (verbose > 4)
1125 : : obatched(clog) << nickname << " bind " << parameter << "=" << "NULL" << endl;
1126 : : int rc = sqlite3_bind_null (this->pp, parameter);
1127 : : if (rc != SQLITE_OK)
1128 : : throw sqlite_exception(rc, "sqlite3 bind");
1129 : : return *this;
1130 : : }
1131 : :
1132 : :
1133 : 31614 : void step_ok_done() {
1134 [ + - + - : 63229 : tmp_ms_metric tick("sqlite3","step_done",nickname);
- + - - ]
1135 [ + - ]: 31615 : int rc = sqlite3_step (this->pp);
1136 [ + + ]: 31620 : if (verbose > 4)
1137 [ + - + - : 832 : obatched(clog) << nickname << " step-ok-done(" << sqlite3_errstr(rc) << ") " << sql << endl;
+ - + - +
- + - + -
+ - ]
1138 [ + + - + ]: 31620 : if (rc != SQLITE_OK && rc != SQLITE_DONE && rc != SQLITE_ROW)
1139 [ # # # # ]: 0 : throw sqlite_exception(rc, "sqlite3 step");
1140 [ + - ]: 31620 : (void) sqlite3_reset (this->pp);
1141 : 31620 : }
1142 : :
1143 : :
1144 : 12581 : int step() {
1145 [ + - + - : 25159 : tmp_ms_metric tick("sqlite3","step",nickname);
- + - - ]
1146 [ + - ]: 12578 : int rc = sqlite3_step (this->pp);
1147 [ + + ]: 12580 : if (verbose > 4)
1148 [ + - + - : 378 : obatched(clog) << nickname << " step(" << sqlite3_errstr(rc) << ") " << sql << endl;
+ - + - +
- + - + -
+ - ]
1149 : 12581 : return rc;
1150 : 12580 : }
1151 : :
1152 [ + + + + ]: 12732 : ~sqlite_ps () { sqlite3_finalize (this->pp); }
1153 [ + - + - : 2580 : operator sqlite3_stmt* () { return this->pp; }
+ - ]
1154 : : };
1155 : :
1156 : :
1157 : : ////////////////////////////////////////////////////////////////////////
1158 : :
1159 : :
1160 : : struct sqlite_checkpoint_pb: public periodic_barrier
1161 : : {
1162 : : // NB: don't use sqlite_ps since it can throw exceptions during ctor etc.
1163 : 64 : sqlite_checkpoint_pb(unsigned t, unsigned p):
1164 : 128 : periodic_barrier(t, p) { }
1165 : :
1166 : 30 : void periodic_barrier_work() noexcept
1167 : : {
1168 : 30 : (void) sqlite3_exec (db, "pragma wal_checkpoint(truncate);", NULL, NULL, NULL);
1169 : 30 : }
1170 : : };
1171 : :
1172 : : static periodic_barrier* scan_barrier = 0; // initialized in main()
1173 : :
1174 : :
1175 : : ////////////////////////////////////////////////////////////////////////
1176 : :
1177 : : // RAII style templated autocloser
1178 : :
1179 : : template <class Payload, class Ignore>
1180 : : struct defer_dtor
1181 : : {
1182 : : public:
1183 : : typedef Ignore (*dtor_fn) (Payload);
1184 : :
1185 : : private:
1186 : : Payload p;
1187 : : dtor_fn fn;
1188 : :
1189 : : public:
1190 : 5343 : defer_dtor(Payload _p, dtor_fn _fn): p(_p), fn(_fn) {}
1191 : 2230 : ~defer_dtor() { (void) (*fn)(p); }
1192 : :
1193 : : private:
1194 : : defer_dtor(const defer_dtor<Payload,Ignore>&); // make uncopyable
1195 : : defer_dtor& operator=(const defer_dtor<Payload,Ignore> &); // make unassignable
1196 : : };
1197 : :
1198 : :
1199 : :
1200 : : ////////////////////////////////////////////////////////////////////////
1201 : :
1202 : :
1203 : : static string
1204 : 4152 : header_censor(const string& str)
1205 : : {
1206 : 4152 : string y;
1207 [ + + ]: 35044 : for (auto&& x : str)
1208 : : {
1209 [ + + ]: 30892 : if (isalnum(x) || x == '/' || x == '.' || x == ',' || x == '_' || x == ':')
1210 [ + - ]: 61778 : y += x;
1211 : : }
1212 : 4152 : return y;
1213 : 0 : }
1214 : :
1215 : :
1216 : : static string
1217 : 2076 : conninfo (struct MHD_Connection * conn)
1218 : : {
1219 : 2076 : char hostname[256]; // RFC1035
1220 : 2076 : char servname[256];
1221 : 2076 : int sts = -1;
1222 : :
1223 [ - + ]: 2076 : if (conn == 0)
1224 : 0 : return "internal";
1225 : :
1226 : : /* Look up client address data. */
1227 : 2076 : const union MHD_ConnectionInfo *u = MHD_get_connection_info (conn,
1228 : : MHD_CONNECTION_INFO_CLIENT_ADDRESS);
1229 [ + - ]: 2076 : struct sockaddr *so = u ? u->client_addr : 0;
1230 : :
1231 [ + - - + ]: 2076 : if (so && so->sa_family == AF_INET) {
1232 : 0 : sts = getnameinfo (so, sizeof (struct sockaddr_in),
1233 : : hostname, sizeof (hostname),
1234 : : servname, sizeof (servname),
1235 : : NI_NUMERICHOST | NI_NUMERICSERV);
1236 [ + - ]: 2076 : } else if (so && so->sa_family == AF_INET6) {
1237 : 2076 : struct sockaddr_in6* addr6 = (struct sockaddr_in6*) so;
1238 [ + - + - : 2076 : if (IN6_IS_ADDR_V4MAPPED(&addr6->sin6_addr)) {
- + ]
1239 : 2076 : struct sockaddr_in addr4;
1240 : 2076 : memset (&addr4, 0, sizeof(addr4));
1241 : 2076 : addr4.sin_family = AF_INET;
1242 : 2076 : addr4.sin_port = addr6->sin6_port;
1243 : 2076 : memcpy (&addr4.sin_addr.s_addr, addr6->sin6_addr.s6_addr+12, sizeof(addr4.sin_addr.s_addr));
1244 : 2076 : sts = getnameinfo ((struct sockaddr*) &addr4, sizeof (addr4),
1245 : : hostname, sizeof (hostname),
1246 : : servname, sizeof (servname),
1247 : : NI_NUMERICHOST | NI_NUMERICSERV);
1248 : : } else {
1249 : 0 : sts = getnameinfo (so, sizeof (struct sockaddr_in6),
1250 : : hostname, sizeof (hostname),
1251 : : servname, sizeof (servname),
1252 : : NI_NUMERICHOST | NI_NUMERICSERV);
1253 : : }
1254 : : }
1255 : :
1256 [ - + ]: 2076 : if (sts != 0) {
1257 : 0 : hostname[0] = servname[0] = '\0';
1258 : : }
1259 : :
1260 : : // extract headers relevant to administration
1261 [ - + ]: 2076 : const char* user_agent = MHD_lookup_connection_value (conn, MHD_HEADER_KIND, "User-Agent") ?: "";
1262 [ + + ]: 2076 : const char* x_forwarded_for = MHD_lookup_connection_value (conn, MHD_HEADER_KIND, "X-Forwarded-For") ?: "";
1263 : : // NB: these are untrustworthy, beware if machine-processing log files
1264 : :
1265 [ + - + - : 6228 : return string(hostname) + string(":") + string(servname) +
+ - + - +
- + - - +
- + - + -
+ - + - +
- - - - -
- ]
1266 [ + - + - : 8606 : string(" UA:") + header_censor(string(user_agent)) +
+ - + - +
- - + + +
- + + + -
+ - - -
- ]
1267 [ + - + - : 6240 : string(" XFF:") + header_censor(string(x_forwarded_for));
+ - + + +
+ - - ]
1268 : : }
1269 : :
1270 : :
1271 : :
1272 : : ////////////////////////////////////////////////////////////////////////
1273 : :
1274 : : /* Wrapper for MHD_add_response_header that logs an error if we
1275 : : couldn't add the specified header. */
1276 : : static void
1277 : 6052 : add_mhd_response_header (struct MHD_Response *r,
1278 : : const char *h, const char *v)
1279 : : {
1280 [ - + ]: 6052 : if (MHD_add_response_header (r, h, v) == MHD_NO)
1281 [ # # # # : 0 : obatched(clog) << "Error: couldn't add '" << h << "' header" << endl;
# # ]
1282 : 6052 : }
1283 : :
1284 : : static void
1285 : 802 : add_mhd_last_modified (struct MHD_Response *resp, time_t mtime)
1286 : : {
1287 : 802 : struct tm now;
1288 : 802 : struct tm *nowp = gmtime_r (&mtime, &now);
1289 [ + - ]: 802 : if (nowp != NULL)
1290 : : {
1291 : 802 : char datebuf[80];
1292 : 802 : size_t rc = strftime (datebuf, sizeof (datebuf), "%a, %d %b %Y %T GMT",
1293 : : nowp);
1294 [ + - ]: 802 : if (rc > 0 && rc < sizeof (datebuf))
1295 : 802 : add_mhd_response_header (resp, "Last-Modified", datebuf);
1296 : : }
1297 : :
1298 : 802 : add_mhd_response_header (resp, "Cache-Control", "public");
1299 : 802 : }
1300 : :
1301 : : // quote all questionable characters of str for safe passage through a sh -c expansion.
1302 : : static string
1303 : 560 : shell_escape(const string& str)
1304 : : {
1305 : 560 : string y;
1306 [ + + ]: 78876 : for (auto&& x : str)
1307 : : {
1308 [ + + + + ]: 78316 : if (! isalnum(x) && x != '/')
1309 [ + - ]: 6622 : y += "\\";
1310 [ + - ]: 156632 : y += x;
1311 : : }
1312 : 560 : return y;
1313 : 0 : }
1314 : :
1315 : :
1316 : : // PR25548: Perform POSIX / RFC3986 style path canonicalization on the input string.
1317 : : //
1318 : : // Namely:
1319 : : // // -> /
1320 : : // /foo/../ -> /
1321 : : // /./ -> /
1322 : : //
1323 : : // This mapping is done on dwarf-side source path names, which may
1324 : : // include these constructs, so we can deal with debuginfod clients
1325 : : // that accidentally canonicalize the paths.
1326 : : //
1327 : : // realpath(3) is close but not quite right, because it also resolves
1328 : : // symbolic links. Symlinks at the debuginfod server have nothing to
1329 : : // do with the build-time symlinks, thus they must not be considered.
1330 : : //
1331 : : // see also curl Curl_dedotdotify() aka RFC3986, which we mostly follow here
1332 : : // see also libc __realpath()
1333 : : // see also llvm llvm::sys::path::remove_dots()
1334 : : static string
1335 : 3552 : canon_pathname (const string& input)
1336 : : {
1337 : 3552 : string i = input; // 5.2.4 (1)
1338 : 3552 : string o;
1339 : :
1340 : 33188 : while (i.size() != 0)
1341 : : {
1342 : : // 5.2.4 (2) A
1343 [ + - - + : 59272 : if (i.substr(0,3) == "../")
- + ]
1344 [ # # # # ]: 0 : i = i.substr(3);
1345 [ + - - + : 59272 : else if(i.substr(0,2) == "./")
- + ]
1346 [ # # # # ]: 0 : i = i.substr(2);
1347 : :
1348 : : // 5.2.4 (2) B
1349 [ + - - + : 59272 : else if (i.substr(0,3) == "/./")
+ + ]
1350 [ + - + + ]: 622 : i = i.substr(2);
1351 [ - + ]: 29254 : else if (i == "/.")
1352 [ # # ]: 0 : i = ""; // no need to handle "/." complete-path-segment case; we're dealing with file names
1353 : :
1354 : : // 5.2.4 (2) C
1355 [ + - - + : 58508 : else if (i.substr(0,4) == "/../") {
+ + ]
1356 [ + - + + ]: 472 : i = i.substr(3);
1357 : 472 : string::size_type sl = o.rfind("/");
1358 [ + - ]: 472 : if (sl != string::npos)
1359 [ + - + - ]: 944 : o = o.substr(0, sl);
1360 : : else
1361 [ # # ]: 0 : o = "";
1362 [ - + ]: 28782 : } else if (i == "/..")
1363 [ # # ]: 0 : i = ""; // no need to handle "/.." complete-path-segment case; we're dealing with file names
1364 : :
1365 : : // 5.2.4 (2) D
1366 : : // no need to handle these cases; we're dealing with file names
1367 [ - + ]: 28782 : else if (i == ".")
1368 [ # # ]: 0 : i = "";
1369 [ - + ]: 28782 : else if (i == "..")
1370 [ # # ]: 0 : i = "";
1371 : :
1372 : : // POSIX special: map // to /
1373 [ + - - + : 57564 : else if (i.substr(0,2) == "//")
+ + ]
1374 [ + - + + ]: 152 : i = i.substr(1);
1375 : :
1376 : : // 5.2.4 (2) E
1377 : : else {
1378 [ - + ]: 28646 : string::size_type next_slash = i.find("/", (i[0]=='/' ? 1 : 0)); // skip first slash
1379 [ + - + + ]: 57292 : o += i.substr(0, next_slash);
1380 [ + + ]: 28646 : if (next_slash == string::npos)
1381 [ + - + + ]: 36740 : i = "";
1382 : : else
1383 [ + - + + : 47296 : i = i.substr(next_slash);
- - ]
1384 : : }
1385 : : }
1386 : :
1387 [ + - ]: 3552 : return o;
1388 : 3552 : }
1389 : :
1390 : :
1391 : : // Estimate available free space for a given filesystem via statfs(2).
1392 : : // Return true if the free fraction is known to be smaller than the
1393 : : // given minimum percentage. Also update a related metric.
1394 : 3909 : bool statfs_free_enough_p(const string& path, const string& label, long minfree = 0)
1395 : : {
1396 : 3909 : struct statfs sfs;
1397 : 3909 : int rc = statfs(path.c_str(), &sfs);
1398 [ + + ]: 3909 : if (rc == 0)
1399 : : {
1400 : 3859 : double s = (double) sfs.f_bavail / (double) sfs.f_blocks;
1401 [ + - + - : 7718 : set_metric("filesys_free_ratio","purpose",label, s);
- + - - ]
1402 : 3859 : return ((s * 100.0) < minfree);
1403 : : }
1404 : : return false;
1405 : : }
1406 : :
1407 : :
1408 : :
1409 : : // A map-like class that owns a cache of file descriptors (indexed by
1410 : : // file / content names).
1411 : : //
1412 : : // If only it could use fd's instead of file names ... but we can't
1413 : : // dup(2) to create independent descriptors for the same unlinked
1414 : : // files, so would have to use some goofy linux /proc/self/fd/%d
1415 : : // hack such as the following
1416 : :
1417 : : #if 0
1418 : : int superdup(int fd)
1419 : : {
1420 : : #ifdef __linux__
1421 : : char *fdpath = NULL;
1422 : : int rc = asprintf(& fdpath, "/proc/self/fd/%d", fd);
1423 : : int newfd;
1424 : : if (rc >= 0)
1425 : : newfd = open(fdpath, O_RDONLY);
1426 : : else
1427 : : newfd = -1;
1428 : : free (fdpath);
1429 : : return newfd;
1430 : : #else
1431 : : return -1;
1432 : : #endif
1433 : : }
1434 : : #endif
1435 : :
1436 : : class libarchive_fdcache
1437 : : {
1438 : : private:
1439 : : mutex fdcache_lock;
1440 : :
1441 : : struct fdcache_entry
1442 : : {
1443 : : string archive;
1444 : : string entry;
1445 : : string fd;
1446 : : double fd_size_mb; // slightly rounded up megabytes
1447 : : };
1448 : : deque<fdcache_entry> lru; // @head: most recently used
1449 : : long max_fds;
1450 : : deque<fdcache_entry> prefetch; // prefetched
1451 : : long max_mbs;
1452 : : long max_prefetch_mbs;
1453 : : long max_prefetch_fds;
1454 : :
1455 : : public:
1456 : 2168 : void set_metrics()
1457 : : {
1458 : 2168 : double fdcache_mb = 0.0;
1459 : 2168 : double prefetch_mb = 0.0;
1460 [ + + + + ]: 7876 : for (auto i = lru.begin(); i < lru.end(); i++)
1461 : 3540 : fdcache_mb += i->fd_size_mb;
1462 [ + + + + ]: 5482 : for (auto j = prefetch.begin(); j < prefetch.end(); j++)
1463 : 1146 : prefetch_mb += j->fd_size_mb;
1464 [ + - ]: 2168 : set_metric("fdcache_bytes", fdcache_mb*1024.0*1024.0);
1465 [ + - ]: 2168 : set_metric("fdcache_count", lru.size());
1466 [ + - ]: 2168 : set_metric("fdcache_prefetch_bytes", prefetch_mb*1024.0*1024.0);
1467 [ + - ]: 2168 : set_metric("fdcache_prefetch_count", prefetch.size());
1468 : 2168 : }
1469 : :
1470 : 1218 : void intern(const string& a, const string& b, string fd, off_t sz, bool front_p)
1471 : : {
1472 : 1218 : {
1473 : 1218 : unique_lock<mutex> lock(fdcache_lock);
1474 : : // nuke preexisting copy
1475 [ + + + + ]: 4788 : for (auto i = lru.begin(); i < lru.end(); i++)
1476 : : {
1477 [ + + - + ]: 1176 : if (i->archive == a && i->entry == b)
1478 : : {
1479 : 0 : unlink (i->fd.c_str());
1480 : 0 : lru.erase(i);
1481 [ # # # # : 0 : inc_metric("fdcache_op_count","op","dequeue");
# # # # #
# # # # #
# # ]
1482 : 0 : break; // must not continue iterating
1483 : : }
1484 : : }
1485 : : // nuke preexisting copy in prefetch
1486 [ + + + + ]: 3536 : for (auto i = prefetch.begin(); i < prefetch.end(); i++)
1487 : : {
1488 [ + + + - ]: 550 : if (i->archive == a && i->entry == b)
1489 : : {
1490 : 0 : unlink (i->fd.c_str());
1491 : 0 : prefetch.erase(i);
1492 [ # # # # : 0 : inc_metric("fdcache_op_count","op","prefetch_dequeue");
# # # # #
# # # # #
# # ]
1493 : 0 : break; // must not continue iterating
1494 : : }
1495 : : }
1496 : 1218 : double mb = (sz+65535)/1048576.0; // round up to 64K block
1497 [ + - + - : 1218 : fdcache_entry n = { a, b, fd, mb };
+ - ]
1498 [ + + ]: 1218 : if (front_p)
1499 : : {
1500 [ + - + - : 1332 : inc_metric("fdcache_op_count","op","enqueue");
+ - + - -
+ - + - -
- - ]
1501 [ + - ]: 666 : lru.push_front(n);
1502 : : }
1503 : : else
1504 : : {
1505 [ + - + - : 1104 : inc_metric("fdcache_op_count","op","prefetch_enqueue");
+ - + - -
+ + - - -
- - ]
1506 [ + - ]: 552 : prefetch.push_front(n);
1507 : : }
1508 [ + + ]: 1218 : if (verbose > 3)
1509 [ + - + - ]: 3564 : obatched(clog) << "fdcache interned a=" << a << " b=" << b
1510 [ + - + - : 1188 : << " fd=" << fd << " mb=" << mb << " front=" << front_p << endl;
+ - + - +
- + - + -
+ - + - ]
1511 : :
1512 [ + - ]: 1218 : set_metrics();
1513 : 1218 : }
1514 : :
1515 : : // NB: we age the cache at lookup time too
1516 [ + - - + : 1218 : if (statfs_free_enough_p(tmpdir, "tmpdir", fdcache_mintmp))
- + ]
1517 : : {
1518 [ # # # # : 0 : inc_metric("fdcache_op_count","op","emerg-flush");
# # # # #
# # # #
# ]
1519 [ # # ]: 0 : obatched(clog) << "fdcache emergency flush for filling tmpdir" << endl;
1520 : 0 : this->limit(0, 0, 0, 0); // emergency flush
1521 : : }
1522 [ + + ]: 1218 : else if (front_p)
1523 : 666 : this->limit(max_fds, max_mbs, max_prefetch_fds, max_prefetch_mbs); // age cache if required
1524 : 1218 : }
1525 : :
1526 : 740 : int lookup(const string& a, const string& b)
1527 : : {
1528 : 740 : int fd = -1;
1529 : 740 : {
1530 : 740 : unique_lock<mutex> lock(fdcache_lock);
1531 [ + + + + ]: 4008 : for (auto i = lru.begin(); i < lru.end(); i++)
1532 : : {
1533 [ + + + + ]: 1320 : if (i->archive == a && i->entry == b)
1534 : : { // found it; move it to head of lru
1535 [ + - ]: 56 : fdcache_entry n = *i;
1536 : 56 : lru.erase(i); // invalidates i, so no more iteration!
1537 [ + - ]: 56 : lru.push_front(n);
1538 [ + - + - : 112 : inc_metric("fdcache_op_count","op","requeue_front");
+ - + - -
+ - + - -
- - ]
1539 [ + - ]: 56 : fd = open(n.fd.c_str(), O_RDONLY);
1540 : 56 : break;
1541 : 56 : }
1542 : : }
1543 : : // Iterate through prefetch while fd == -1 to ensure that no duplication between lru and
1544 : : // prefetch occurs.
1545 [ + + + + : 2492 : for ( auto i = prefetch.begin(); fd == -1 && i < prefetch.end(); ++i)
+ + ]
1546 : : {
1547 [ + + + + ]: 548 : if (i->archive == a && i->entry == b)
1548 : : { // found it; take the entry from the prefetch deque to the lru deque, since it has now been accessed.
1549 [ + - ]: 14 : fdcache_entry n = *i;
1550 : 14 : prefetch.erase(i);
1551 [ + - ]: 14 : lru.push_front(n);
1552 [ + - + - : 28 : inc_metric("fdcache_op_count","op","prefetch_access");
+ - + - -
+ - + - -
- - ]
1553 [ + - ]: 14 : fd = open(n.fd.c_str(), O_RDONLY);
1554 : 14 : break;
1555 : 14 : }
1556 : : }
1557 : 0 : }
1558 : :
1559 [ + - - + : 740 : if (statfs_free_enough_p(tmpdir, "tmpdir", fdcache_mintmp))
- + ]
1560 : : {
1561 [ # # # # : 0 : inc_metric("fdcache_op_count","op","emerg-flush");
# # # # #
# # # #
# ]
1562 [ # # ]: 0 : obatched(clog) << "fdcache emergency flush for filling tmpdir" << endl;
1563 : 0 : this->limit(0, 0, 0, 0); // emergency flush
1564 : : }
1565 [ + + ]: 740 : else if (fd >= 0)
1566 : 70 : this->limit(max_fds, max_mbs, max_prefetch_fds, max_prefetch_mbs); // age cache if required
1567 : :
1568 : 740 : return fd;
1569 : : }
1570 : :
1571 : 1236 : int probe(const string& a, const string& b) // just a cache residency check - don't modify LRU state, don't open
1572 : : {
1573 : 1236 : unique_lock<mutex> lock(fdcache_lock);
1574 [ + + + + ]: 4948 : for (auto i = lru.begin(); i < lru.end(); i++)
1575 : : {
1576 [ + + + + ]: 1264 : if (i->archive == a && i->entry == b)
1577 : : {
1578 [ + - + - : 52 : inc_metric("fdcache_op_count","op","probe_hit");
+ - + - -
+ - + - -
- - ]
1579 : 26 : return true;
1580 : : }
1581 : : }
1582 [ + + + + ]: 3512 : for (auto i = prefetch.begin(); i < prefetch.end(); i++)
1583 : : {
1584 [ + + + - ]: 546 : if (i->archive == a && i->entry == b)
1585 : : {
1586 [ # # # # : 0 : inc_metric("fdcache_op_count","op","prefetch_probe_hit");
# # # # #
# # # # #
# # ]
1587 : 0 : return true;
1588 : : }
1589 : : }
1590 [ + - + - : 2420 : inc_metric("fdcache_op_count","op","probe_miss");
+ - + - -
+ - + - -
- - ]
1591 : 1210 : return false;
1592 : 1236 : }
1593 : :
1594 : 0 : void clear(const string& a, const string& b)
1595 : : {
1596 : 0 : unique_lock<mutex> lock(fdcache_lock);
1597 [ # # # # ]: 0 : for (auto i = lru.begin(); i < lru.end(); i++)
1598 : : {
1599 [ # # # # ]: 0 : if (i->archive == a && i->entry == b)
1600 : : { // found it; erase it from lru
1601 [ # # ]: 0 : fdcache_entry n = *i;
1602 : 0 : lru.erase(i); // invalidates i, so no more iteration!
1603 [ # # # # : 0 : inc_metric("fdcache_op_count","op","clear");
# # # # #
# # # # #
# # ]
1604 : 0 : unlink (n.fd.c_str());
1605 [ # # ]: 0 : set_metrics();
1606 : 0 : return;
1607 : 0 : }
1608 : : }
1609 [ # # # # ]: 0 : for (auto i = prefetch.begin(); i < prefetch.end(); i++)
1610 : : {
1611 [ # # # # ]: 0 : if (i->archive == a && i->entry == b)
1612 : : { // found it; erase it from lru
1613 [ # # ]: 0 : fdcache_entry n = *i;
1614 : 0 : prefetch.erase(i); // invalidates i, so no more iteration!
1615 [ # # # # : 0 : inc_metric("fdcache_op_count","op","prefetch_clear");
# # # # #
# # # # #
# # ]
1616 : 0 : unlink (n.fd.c_str());
1617 [ # # ]: 0 : set_metrics();
1618 : 0 : return;
1619 : 0 : }
1620 : : }
1621 : 0 : }
1622 : :
1623 : 1022 : void limit(long maxfds, long maxmbs, long maxprefetchfds, long maxprefetchmbs , bool metrics_p = true)
1624 : : {
1625 [ + + + + : 1022 : if (verbose > 3 && (this->max_fds != maxfds || this->max_mbs != maxmbs))
+ + ]
1626 [ + - + - : 376 : obatched(clog) << "fdcache limited to maxfds=" << maxfds << " maxmbs=" << maxmbs << endl;
+ - + - ]
1627 : :
1628 : 1022 : unique_lock<mutex> lock(fdcache_lock);
1629 : 1022 : this->max_fds = maxfds;
1630 : 1022 : this->max_mbs = maxmbs;
1631 : 1022 : this->max_prefetch_fds = maxprefetchfds;
1632 : 1022 : this->max_prefetch_mbs = maxprefetchmbs;
1633 : 1022 : long total_fd = 0;
1634 : 1022 : double total_mb = 0.0;
1635 [ + + + + ]: 5440 : for (auto i = lru.begin(); i < lru.end(); i++)
1636 : : {
1637 : : // accumulate totals from most recently used one going backward
1638 : 2240 : total_fd ++;
1639 [ + + ]: 2240 : total_mb += i->fd_size_mb;
1640 [ + + + + ]: 2240 : if (total_fd > this->max_fds || total_mb > this->max_mbs)
1641 : : {
1642 : : // found the cut here point!
1643 : :
1644 [ + + + + ]: 2444 : for (auto j = i; j < lru.end(); j++) // close all the fds from here on in
1645 : : {
1646 [ + + ]: 680 : if (verbose > 3)
1647 [ + - + - ]: 2004 : obatched(clog) << "fdcache evicted a=" << j->archive << " b=" << j->entry
1648 [ + - + - : 668 : << " fd=" << j->fd << " mb=" << j->fd_size_mb << endl;
+ - + - +
- + - +
- ]
1649 [ + + ]: 680 : if (metrics_p)
1650 [ + - + - : 1120 : inc_metric("fdcache_op_count","op","evict");
+ - + - -
+ - + - -
- - ]
1651 : 680 : unlink (j->fd.c_str());
1652 : : }
1653 : :
1654 : 542 : lru.erase(i, lru.end()); // erase the nodes generally
1655 : 542 : break;
1656 : : }
1657 : : }
1658 : 1022 : total_fd = 0;
1659 : 1022 : total_mb = 0.0;
1660 [ + + + + ]: 2132 : for(auto i = prefetch.begin(); i < prefetch.end(); i++){
1661 : : // accumulate totals from most recently used one going backward
1662 : 568 : total_fd ++;
1663 [ + + ]: 568 : total_mb += i->fd_size_mb;
1664 [ + + + + ]: 568 : if (total_fd > this->max_prefetch_fds || total_mb > this->max_prefetch_mbs)
1665 : : {
1666 : : // found the cut here point!
1667 [ + + + + ]: 2124 : for (auto j = i; j < prefetch.end(); j++) // close all the fds from here on in
1668 : : {
1669 [ + + ]: 538 : if (verbose > 3)
1670 [ + - + - ]: 1560 : obatched(clog) << "fdcache evicted from prefetch a=" << j->archive << " b=" << j->entry
1671 [ + - + - : 520 : << " fd=" << j->fd << " mb=" << j->fd_size_mb << endl;
+ - + - +
- + - +
- ]
1672 [ + + ]: 538 : if (metrics_p)
1673 [ + - + - : 1012 : inc_metric("fdcache_op_count","op","prefetch_evict");
+ - + - -
+ - + - -
- - - - ]
1674 : 538 : unlink (j->fd.c_str());
1675 : : }
1676 : :
1677 : 524 : prefetch.erase(i, prefetch.end()); // erase the nodes generally
1678 : 524 : break;
1679 : : }
1680 : : }
1681 [ + + + - ]: 1022 : if (metrics_p) set_metrics();
1682 : 1022 : }
1683 : :
1684 : :
1685 : 72 : ~libarchive_fdcache()
1686 : : {
1687 : : // unlink any fdcache entries in $TMPDIR
1688 : : // don't update metrics; those globals may be already destroyed
1689 : 72 : limit(0, 0, 0, 0, false);
1690 : 72 : }
1691 : : };
1692 : : static libarchive_fdcache fdcache;
1693 : :
1694 : : /* Search ELF_FD for an ELF/DWARF section with name SECTION.
1695 : : If found copy the section to a temporary file and return
1696 : : its file descriptor, otherwise return -1.
1697 : :
1698 : : The temporary file's mtime will be set to PARENT_MTIME.
1699 : : B_SOURCE should be a description of the parent file suitable
1700 : : for printing to the log. */
1701 : :
1702 : : static int
1703 : 12 : extract_section (int elf_fd, int64_t parent_mtime,
1704 : : const string& b_source, const string& section)
1705 : : {
1706 : : /* Search the fdcache. */
1707 : 12 : struct stat fs;
1708 : 12 : int fd = fdcache.lookup (b_source, section);
1709 [ - + ]: 12 : if (fd >= 0)
1710 : : {
1711 [ # # ]: 0 : if (fstat (fd, &fs) != 0)
1712 : : {
1713 [ # # ]: 0 : if (verbose)
1714 [ # # ]: 0 : obatched (clog) << "cannot fstate fdcache "
1715 [ # # # # : 0 : << b_source << " " << section << endl;
# # ]
1716 : 0 : close (fd);
1717 : 0 : return -1;
1718 : : }
1719 [ # # ]: 0 : if ((int64_t) fs.st_mtime != parent_mtime)
1720 : : {
1721 [ # # ]: 0 : if (verbose)
1722 [ # # ]: 0 : obatched(clog) << "mtime mismatch for "
1723 [ # # # # : 0 : << b_source << " " << section << endl;
# # ]
1724 : 0 : close (fd);
1725 : 0 : return -1;
1726 : : }
1727 : : /* Success. */
1728 : : return fd;
1729 : : }
1730 : :
1731 : 12 : Elf *elf = elf_begin (elf_fd, ELF_C_READ_MMAP_PRIVATE, NULL);
1732 [ - + ]: 12 : if (elf == NULL)
1733 : : return -1;
1734 : :
1735 : : /* Try to find the section and copy the contents into a separate file. */
1736 : 12 : try
1737 : : {
1738 : 12 : size_t shstrndx;
1739 [ + - ]: 12 : int rc = elf_getshdrstrndx (elf, &shstrndx);
1740 [ - + ]: 12 : if (rc < 0)
1741 [ # # # # ]: 0 : throw elfutils_exception (rc, "getshdrstrndx");
1742 : :
1743 : : Elf_Scn *scn = NULL;
1744 : 432 : while (true)
1745 : : {
1746 [ + - ]: 222 : scn = elf_nextscn (elf, scn);
1747 [ + - ]: 222 : if (scn == NULL)
1748 : : break;
1749 : 222 : GElf_Shdr shdr_storage;
1750 [ + - ]: 222 : GElf_Shdr *shdr = gelf_getshdr (scn, &shdr_storage);
1751 [ + - ]: 222 : if (shdr == NULL)
1752 : : break;
1753 : :
1754 [ + - ]: 222 : const char *scn_name = elf_strptr (elf, shstrndx, shdr->sh_name);
1755 [ + - ]: 222 : if (scn_name == NULL)
1756 : : break;
1757 [ + + ]: 222 : if (scn_name == section)
1758 : : {
1759 : 12 : Elf_Data *data = NULL;
1760 : :
1761 : : /* We found the desired section. */
1762 [ + - ]: 12 : data = elf_rawdata (scn, NULL);
1763 [ - + ]: 12 : if (data == NULL)
1764 [ # # # # : 0 : throw elfutils_exception (elf_errno (), "elfraw_data");
# # ]
1765 [ + + ]: 12 : if (data->d_buf == NULL)
1766 : : {
1767 [ + - + - ]: 8 : obatched(clog) << "section " << section
1768 [ + - + - ]: 4 : << " is empty" << endl;
1769 : 4 : break;
1770 : : }
1771 : :
1772 : : /* Create temporary file containing the section. */
1773 : 8 : char *tmppath = NULL;
1774 : 8 : rc = asprintf (&tmppath, "%s/debuginfod.XXXXXX", tmpdir.c_str());
1775 [ - + ]: 8 : if (rc < 0)
1776 [ # # # # ]: 0 : throw libc_exception (ENOMEM, "cannot allocate tmppath");
1777 : 8 : defer_dtor<void*,void> tmmpath_freer (tmppath, free);
1778 [ + - ]: 8 : fd = mkstemp (tmppath);
1779 [ - + ]: 8 : if (fd < 0)
1780 [ # # # # ]: 0 : throw libc_exception (errno, "cannot create temporary file");
1781 [ + - ]: 8 : ssize_t res = write_retry (fd, data->d_buf, data->d_size);
1782 [ + - - + ]: 8 : if (res < 0 || (size_t) res != data->d_size)
1783 [ # # # # ]: 0 : throw libc_exception (errno, "cannot write to temporary file");
1784 : :
1785 : : /* Set mtime to be the same as the parent file's mtime. */
1786 : 8 : struct timespec tvs[2];
1787 [ - + ]: 8 : if (fstat (elf_fd, &fs) != 0)
1788 [ # # # # ]: 0 : throw libc_exception (errno, "cannot fstat file");
1789 : :
1790 : 8 : tvs[0].tv_sec = 0;
1791 : 8 : tvs[0].tv_nsec = UTIME_OMIT;
1792 : 8 : tvs[1] = fs.st_mtim;
1793 : 8 : (void) futimens (fd, tvs);
1794 : :
1795 : : /* Add to fdcache. */
1796 [ + - + - ]: 8 : fdcache.intern (b_source, section, tmppath, data->d_size, true);
1797 : 8 : break;
1798 : 12 : }
1799 : 210 : }
1800 : : }
1801 [ - - ]: 0 : catch (const reportable_exception &e)
1802 : : {
1803 [ - - ]: 0 : e.report (clog);
1804 [ - - ]: 0 : close (fd);
1805 : 0 : fd = -1;
1806 : 0 : }
1807 : :
1808 : 12 : elf_end (elf);
1809 : : return fd;
1810 : : }
1811 : :
1812 : : static struct MHD_Response*
1813 : 74 : handle_buildid_f_match (bool internal_req_t,
1814 : : int64_t b_mtime,
1815 : : const string& b_source0,
1816 : : const string& section,
1817 : : int *result_fd)
1818 : : {
1819 : 74 : (void) internal_req_t; // ignored
1820 : 74 : int fd = open(b_source0.c_str(), O_RDONLY);
1821 [ - + ]: 74 : if (fd < 0)
1822 [ # # # # : 0 : throw libc_exception (errno, string("open ") + b_source0);
# # # # ]
1823 : :
1824 : : // NB: use manual close(2) in error case instead of defer_dtor, because
1825 : : // in the normal case, we want to hand the fd over to libmicrohttpd for
1826 : : // file transfer.
1827 : :
1828 : 74 : struct stat s;
1829 : 74 : int rc = fstat(fd, &s);
1830 [ - + ]: 74 : if (rc < 0)
1831 : : {
1832 : 0 : close(fd);
1833 [ # # # # : 0 : throw libc_exception (errno, string("fstat ") + b_source0);
# # # # ]
1834 : : }
1835 : :
1836 [ - + ]: 74 : if ((int64_t) s.st_mtime != b_mtime)
1837 : : {
1838 [ # # ]: 0 : if (verbose)
1839 [ # # # # ]: 0 : obatched(clog) << "mtime mismatch for " << b_source0 << endl;
1840 : 0 : close(fd);
1841 : 0 : return 0;
1842 : : }
1843 : :
1844 [ + + ]: 74 : if (!section.empty ())
1845 : : {
1846 : 6 : int scn_fd = extract_section (fd, s.st_mtime, b_source0, section);
1847 : 6 : close (fd);
1848 : :
1849 [ + + ]: 6 : if (scn_fd >= 0)
1850 : 4 : fd = scn_fd;
1851 : : else
1852 : : {
1853 [ + - ]: 2 : if (verbose)
1854 [ + - ]: 6 : obatched (clog) << "cannot find section " << section
1855 [ + - + - : 2 : << " for " << b_source0 << endl;
+ - ]
1856 : 2 : return 0;
1857 : : }
1858 : :
1859 : 4 : rc = fstat(fd, &s);
1860 [ - + ]: 4 : if (rc < 0)
1861 : : {
1862 : 0 : close (fd);
1863 [ # # # # : 0 : throw libc_exception (errno, string ("fstat ") + b_source0
# # # # #
# # # #
# ]
1864 [ # # # # : 0 : + string (" ") + section);
# # # # #
# ]
1865 : : }
1866 : : }
1867 : :
1868 : 72 : struct MHD_Response* r = MHD_create_response_from_fd ((uint64_t) s.st_size, fd);
1869 [ + - + - : 144 : inc_metric ("http_responses_total","result","file");
+ - - + -
+ - - -
- ]
1870 [ - + ]: 72 : if (r == 0)
1871 : : {
1872 [ # # ]: 0 : if (verbose)
1873 [ # # ]: 0 : obatched(clog) << "cannot create fd-response for " << b_source0
1874 [ # # # # : 0 : << " section=" << section << endl;
# # ]
1875 : 0 : close(fd);
1876 : : }
1877 : : else
1878 : : {
1879 : 72 : std::string file = b_source0.substr(b_source0.find_last_of("/")+1, b_source0.length());
1880 [ + - ]: 72 : add_mhd_response_header (r, "Content-Type", "application/octet-stream");
1881 [ + - ]: 72 : add_mhd_response_header (r, "X-DEBUGINFOD-SIZE",
1882 [ + - ]: 72 : to_string(s.st_size).c_str());
1883 [ + - ]: 72 : add_mhd_response_header (r, "X-DEBUGINFOD-FILE", file.c_str());
1884 [ + - ]: 72 : add_mhd_last_modified (r, s.st_mtime);
1885 [ + - ]: 72 : if (verbose > 1)
1886 [ + - + - : 144 : obatched(clog) << "serving file " << b_source0 << " section=" << section << endl;
+ - + - +
- - - ]
1887 : : /* libmicrohttpd will close it. */
1888 [ + - ]: 72 : if (result_fd)
1889 : 72 : *result_fd = fd;
1890 : 72 : }
1891 : :
1892 : : return r;
1893 : : }
1894 : :
1895 : : // For security/portability reasons, many distro-package archives have
1896 : : // a "./" in front of path names; others have nothing, others have
1897 : : // "/". Canonicalize them all to a single leading "/", with the
1898 : : // assumption that this matches the dwarf-derived file names too.
1899 : 3502 : string canonicalized_archive_entry_pathname(struct archive_entry *e)
1900 : : {
1901 : 3502 : string fn = archive_entry_pathname(e);
1902 [ - + ]: 3502 : if (fn.size() == 0)
1903 : 0 : return fn;
1904 [ - + ]: 3502 : if (fn[0] == '/')
1905 : 0 : return fn;
1906 [ + + ]: 3502 : if (fn[0] == '.')
1907 [ + - ]: 2296 : return fn.substr(1);
1908 : : else
1909 [ + - + - : 2412 : return string("/")+fn;
- - ]
1910 : 3502 : }
1911 : :
1912 : :
1913 : :
1914 : : static struct MHD_Response*
1915 : 786 : handle_buildid_r_match (bool internal_req_p,
1916 : : int64_t b_mtime,
1917 : : const string& b_source0,
1918 : : const string& b_source1,
1919 : : const string& section,
1920 : : int *result_fd)
1921 : : {
1922 : 786 : struct stat fs;
1923 : 786 : int rc = stat (b_source0.c_str(), &fs);
1924 [ + + ]: 786 : if (rc != 0)
1925 [ + - + - : 116 : throw libc_exception (errno, string("stat ") + b_source0);
+ - - + ]
1926 : :
1927 [ - + ]: 728 : if ((int64_t) fs.st_mtime != b_mtime)
1928 : : {
1929 [ # # ]: 0 : if (verbose)
1930 [ # # # # ]: 0 : obatched(clog) << "mtime mismatch for " << b_source0 << endl;
1931 : 0 : return 0;
1932 : : }
1933 : :
1934 : : // check for a match in the fdcache first
1935 : 728 : int fd = fdcache.lookup(b_source0, b_source1);
1936 [ + + ]: 728 : while (fd >= 0) // got one!; NB: this is really an if() with a possible branch out to the end
1937 : : {
1938 : 70 : rc = fstat(fd, &fs);
1939 [ - + ]: 70 : if (rc < 0) // disappeared?
1940 : : {
1941 [ # # ]: 0 : if (verbose)
1942 [ # # # # ]: 0 : obatched(clog) << "cannot fstat fdcache " << b_source0 << endl;
1943 : 0 : close(fd);
1944 : 0 : fdcache.clear(b_source0, b_source1);
1945 : : break; // branch out of if "loop", to try new libarchive fetch attempt
1946 : : }
1947 : :
1948 [ + + ]: 70 : if (!section.empty ())
1949 : : {
1950 [ + - + - ]: 2 : int scn_fd = extract_section (fd, fs.st_mtime,
1951 [ + - - + ]: 4 : b_source0 + ":" + b_source1,
1952 : : section);
1953 : 2 : close (fd);
1954 [ - + ]: 2 : if (scn_fd >= 0)
1955 : 0 : fd = scn_fd;
1956 : : else
1957 : : {
1958 [ + - ]: 2 : if (verbose)
1959 [ + - ]: 6 : obatched (clog) << "cannot find section " << section
1960 : : << " for archive " << b_source0
1961 [ + - + - : 2 : << " file " << b_source1 << endl;
+ - + - +
- ]
1962 : 2 : return 0;
1963 : : }
1964 : :
1965 : 0 : rc = fstat(fd, &fs);
1966 [ # # ]: 0 : if (rc < 0)
1967 : : {
1968 : 0 : close (fd);
1969 [ # # # # ]: 0 : throw libc_exception (errno,
1970 [ # # # # : 0 : string ("fstat archive ") + b_source0 + string (" file ") + b_source1
# # # # #
# # # # #
# # # # #
# # # #
# ]
1971 [ # # # # : 0 : + string (" section ") + section);
# # # # #
# ]
1972 : : }
1973 : : }
1974 : :
1975 : 68 : struct MHD_Response* r = MHD_create_response_from_fd (fs.st_size, fd);
1976 [ - + ]: 68 : if (r == 0)
1977 : : {
1978 [ # # ]: 0 : if (verbose)
1979 [ # # # # ]: 0 : obatched(clog) << "cannot create fd-response for " << b_source0 << endl;
1980 : 0 : close(fd);
1981 : : break; // branch out of if "loop", to try new libarchive fetch attempt
1982 : : }
1983 : :
1984 [ + - + - : 136 : inc_metric ("http_responses_total","result","archive fdcache");
+ - - + -
+ - - -
- ]
1985 : :
1986 : 68 : add_mhd_response_header (r, "Content-Type", "application/octet-stream");
1987 [ + - ]: 68 : add_mhd_response_header (r, "X-DEBUGINFOD-SIZE",
1988 : 68 : to_string(fs.st_size).c_str());
1989 : 68 : add_mhd_response_header (r, "X-DEBUGINFOD-ARCHIVE", b_source0.c_str());
1990 : 68 : add_mhd_response_header (r, "X-DEBUGINFOD-FILE", b_source1.c_str());
1991 : 68 : add_mhd_last_modified (r, fs.st_mtime);
1992 [ + - ]: 68 : if (verbose > 1)
1993 [ + - ]: 204 : obatched(clog) << "serving fdcache archive " << b_source0
1994 : : << " file " << b_source1
1995 [ + - + - : 68 : << " section=" << section << endl;
+ - + - +
- ]
1996 : : /* libmicrohttpd will close it. */
1997 [ + - ]: 68 : if (result_fd)
1998 : 68 : *result_fd = fd;
1999 : : return r;
2000 : : // NB: see, we never go around the 'loop' more than once
2001 : : }
2002 : :
2003 : : // no match ... grumble, must process the archive
2004 : 658 : string archive_decoder = "/dev/null";
2005 [ + - - - ]: 658 : string archive_extension = "";
2006 [ + + ]: 1858 : for (auto&& arch : scan_archives)
2007 [ + + ]: 1200 : if (string_endswith(b_source0, arch.first))
2008 : : {
2009 [ + - ]: 658 : archive_extension = arch.first;
2010 [ + - ]: 1858 : archive_decoder = arch.second;
2011 : : }
2012 : 658 : FILE* fp;
2013 : 658 : defer_dtor<FILE*,int>::dtor_fn dfn;
2014 [ + + ]: 658 : if (archive_decoder != "cat")
2015 : : {
2016 [ + - + - : 1056 : string popen_cmd = archive_decoder + " " + shell_escape(b_source0);
+ - - + -
- - - ]
2017 [ + - ]: 528 : fp = popen (popen_cmd.c_str(), "r"); // "e" O_CLOEXEC?
2018 : 528 : dfn = pclose;
2019 [ - + ]: 528 : if (fp == NULL)
2020 [ # # # # : 0 : throw libc_exception (errno, string("popen ") + popen_cmd);
# # # # ]
2021 : 528 : }
2022 : : else
2023 : : {
2024 [ + - ]: 130 : fp = fopen (b_source0.c_str(), "r");
2025 : 130 : dfn = fclose;
2026 [ - + ]: 130 : if (fp == NULL)
2027 [ # # # # : 0 : throw libc_exception (errno, string("fopen ") + b_source0);
# # # # ]
2028 : : }
2029 : 658 : defer_dtor<FILE*,int> fp_closer (fp, dfn);
2030 : :
2031 : 658 : struct archive *a;
2032 [ + - ]: 658 : a = archive_read_new();
2033 [ - + ]: 658 : if (a == NULL)
2034 [ # # # # ]: 0 : throw archive_exception("cannot create archive reader");
2035 : 658 : defer_dtor<struct archive*,int> archive_closer (a, archive_read_free);
2036 : :
2037 [ + - ]: 658 : rc = archive_read_support_format_all(a);
2038 [ - + ]: 658 : if (rc != ARCHIVE_OK)
2039 [ # # # # ]: 0 : throw archive_exception(a, "cannot select all format");
2040 [ + - ]: 658 : rc = archive_read_support_filter_all(a);
2041 [ - + ]: 658 : if (rc != ARCHIVE_OK)
2042 [ # # # # ]: 0 : throw archive_exception(a, "cannot select all filters");
2043 : :
2044 [ + - ]: 658 : rc = archive_read_open_FILE (a, fp);
2045 [ - + ]: 658 : if (rc != ARCHIVE_OK)
2046 : : {
2047 [ # # # # : 0 : obatched(clog) << "cannot open archive from pipe " << b_source0 << endl;
# # ]
2048 [ # # # # ]: 0 : throw archive_exception(a, "cannot open archive from pipe");
2049 : : }
2050 : :
2051 : : // archive traversal is in three stages, no, four stages:
2052 : : // 1) skip entries whose names do not match the requested one
2053 : : // 2) extract the matching entry name (set r = result)
2054 : : // 3) extract some number of prefetched entries (just into fdcache)
2055 : : // 4) abort any further processing
2056 : 658 : struct MHD_Response* r = 0; // will set in stage 2
2057 [ + + ]: 658 : unsigned prefetch_count =
2058 : : internal_req_p ? 0 : fdcache_prefetch; // will decrement in stage 3
2059 : :
2060 [ + + ]: 10144 : while(r == 0 || prefetch_count > 0) // stage 1, 2, or 3
2061 : : {
2062 [ + - ]: 10124 : if (interrupted)
2063 : : break;
2064 : :
2065 : 10124 : struct archive_entry *e;
2066 [ + - ]: 10124 : rc = archive_read_next_header (a, &e);
2067 [ + + ]: 10124 : if (rc != ARCHIVE_OK)
2068 : : break;
2069 : :
2070 [ + - + + ]: 9486 : if (! S_ISREG(archive_entry_mode (e))) // skip non-files completely
2071 : 6658 : continue;
2072 : :
2073 [ + - ]: 2828 : string fn = canonicalized_archive_entry_pathname (e);
2074 [ + + + + ]: 2828 : if ((r == 0) && (fn != b_source1)) // stage 1
2075 : 1594 : continue;
2076 : :
2077 [ + - + + ]: 1234 : if (fdcache.probe (b_source0, fn) && // skip if already interned
2078 [ - + ]: 24 : fn != b_source1) // but only if we'd just be prefetching, PR29474
2079 : 24 : continue;
2080 : :
2081 : : // extract this file to a temporary file
2082 : 1210 : char* tmppath = NULL;
2083 : 1210 : rc = asprintf (&tmppath, "%s/debuginfod.XXXXXX", tmpdir.c_str());
2084 [ - + ]: 1210 : if (rc < 0)
2085 [ # # # # ]: 0 : throw libc_exception (ENOMEM, "cannot allocate tmppath");
2086 : 1210 : defer_dtor<void*,void> tmmpath_freer (tmppath, free);
2087 [ + - ]: 1210 : fd = mkstemp (tmppath);
2088 [ - + ]: 1210 : if (fd < 0)
2089 [ # # # # ]: 0 : throw libc_exception (errno, "cannot create temporary file");
2090 : : // NB: don't unlink (tmppath), as fdcache will take charge of it.
2091 : :
2092 : : // NB: this can take many uninterruptible seconds for a huge file
2093 [ + - ]: 1210 : rc = archive_read_data_into_fd (a, fd);
2094 [ - + ]: 1210 : if (rc != ARCHIVE_OK) // e.g. ENOSPC!
2095 : : {
2096 [ # # ]: 0 : close (fd);
2097 : 0 : unlink (tmppath);
2098 [ # # # # ]: 0 : throw archive_exception(a, "cannot extract file");
2099 : : }
2100 : :
2101 : : // Set the mtime so the fdcache file mtimes, even prefetched ones,
2102 : : // propagate to future webapi clients.
2103 : 1210 : struct timespec tvs[2];
2104 : 1210 : tvs[0].tv_sec = 0;
2105 : 1210 : tvs[0].tv_nsec = UTIME_OMIT;
2106 [ + - ]: 1210 : tvs[1].tv_sec = archive_entry_mtime(e);
2107 [ + - ]: 1210 : tvs[1].tv_nsec = archive_entry_mtime_nsec(e);
2108 : 1210 : (void) futimens (fd, tvs); /* best effort */
2109 : :
2110 [ + + ]: 1210 : if (r != 0) // stage 3
2111 : : {
2112 : : // NB: now we know we have a complete reusable file; make fdcache
2113 : : // responsible for unlinking it later.
2114 [ + - + - : 552 : fdcache.intern(b_source0, fn,
+ - ]
2115 : : tmppath, archive_entry_size(e),
2116 : : false); // prefetched ones go to the prefetch cache
2117 : 552 : prefetch_count --;
2118 [ + - ]: 552 : close (fd); // we're not saving this fd to make a mhd-response from!
2119 : 552 : continue;
2120 : : }
2121 : :
2122 : : // NB: now we know we have a complete reusable file; make fdcache
2123 : : // responsible for unlinking it later.
2124 [ + - + - : 658 : fdcache.intern(b_source0, b_source1,
+ - ]
2125 : : tmppath, archive_entry_size(e),
2126 : : true); // requested ones go to the front of lru
2127 : :
2128 [ + + ]: 658 : if (!section.empty ())
2129 : : {
2130 [ + - + - ]: 4 : int scn_fd = extract_section (fd, b_mtime,
2131 [ + - + - : 8 : b_source0 + ":" + b_source1,
- + ]
2132 : : section);
2133 [ + - ]: 4 : close (fd);
2134 [ + - ]: 4 : if (scn_fd >= 0)
2135 : 4 : fd = scn_fd;
2136 : : else
2137 : : {
2138 [ # # ]: 0 : if (verbose)
2139 [ # # # # ]: 0 : obatched (clog) << "cannot find section " << section
2140 : : << " for archive " << b_source0
2141 [ # # # # : 0 : << " file " << b_source1 << endl;
# # # # #
# ]
2142 : 0 : return 0;
2143 : : }
2144 : :
2145 : 4 : rc = fstat(fd, &fs);
2146 [ - + ]: 4 : if (rc < 0)
2147 : : {
2148 [ # # ]: 0 : close (fd);
2149 [ # # # # ]: 0 : throw libc_exception (errno,
2150 [ # # # # : 0 : string ("fstat ") + b_source0 + string (" ") + section);
# # # # #
# # # # #
# # # # #
# ]
2151 : : }
2152 [ + - ]: 4 : r = MHD_create_response_from_fd (fs.st_size, fd);
2153 : : }
2154 : : else
2155 [ + - + - ]: 654 : r = MHD_create_response_from_fd (archive_entry_size(e), fd);
2156 : :
2157 [ + - + - : 1316 : inc_metric ("http_responses_total","result",archive_extension + " archive");
+ - + - -
+ + + - -
- - ]
2158 [ - + ]: 658 : if (r == 0)
2159 : : {
2160 [ # # ]: 0 : if (verbose)
2161 [ # # # # : 0 : obatched(clog) << "cannot create fd-response for " << b_source0 << endl;
# # ]
2162 [ # # ]: 0 : close(fd);
2163 : 0 : break; // assume no chance of better luck around another iteration; no other copies of same file
2164 : : }
2165 : : else
2166 : : {
2167 [ + - ]: 658 : std::string file = b_source1.substr(b_source1.find_last_of("/")+1, b_source1.length());
2168 [ + - ]: 658 : add_mhd_response_header (r, "Content-Type",
2169 : : "application/octet-stream");
2170 [ + - ]: 658 : add_mhd_response_header (r, "X-DEBUGINFOD-SIZE",
2171 [ + - + - ]: 658 : to_string(archive_entry_size(e)).c_str());
2172 [ + - ]: 658 : add_mhd_response_header (r, "X-DEBUGINFOD-ARCHIVE",
2173 : : b_source0.c_str());
2174 [ + - ]: 658 : add_mhd_response_header (r, "X-DEBUGINFOD-FILE", file.c_str());
2175 [ + - + - ]: 658 : add_mhd_last_modified (r, archive_entry_mtime(e));
2176 [ + - ]: 658 : if (verbose > 1)
2177 [ + - + - : 1974 : obatched(clog) << "serving archive " << b_source0
- - ]
2178 : : << " file " << b_source1
2179 [ + - + - : 658 : << " section=" << section << endl;
+ - + - +
- ]
2180 : : /* libmicrohttpd will close it. */
2181 [ + - ]: 658 : if (result_fd)
2182 : 658 : *result_fd = fd;
2183 [ + + ]: 658 : continue;
2184 : 658 : }
2185 [ - - - - : 12952 : }
+ + ]
2186 : :
2187 : : // XXX: rpm/file not found: delete this R entry?
2188 : : return r;
2189 [ - + + + ]: 1386 : }
2190 : :
2191 : :
2192 : : static struct MHD_Response*
2193 : 860 : handle_buildid_match (bool internal_req_p,
2194 : : int64_t b_mtime,
2195 : : const string& b_stype,
2196 : : const string& b_source0,
2197 : : const string& b_source1,
2198 : : const string& section,
2199 : : int *result_fd)
2200 : : {
2201 : 860 : try
2202 : : {
2203 [ + + ]: 860 : if (b_stype == "F")
2204 [ + - ]: 74 : return handle_buildid_f_match(internal_req_p, b_mtime, b_source0,
2205 : : section, result_fd);
2206 [ + - ]: 786 : else if (b_stype == "R")
2207 [ + + ]: 786 : return handle_buildid_r_match(internal_req_p, b_mtime, b_source0,
2208 : : b_source1, section, result_fd);
2209 : : }
2210 [ - + ]: 58 : catch (const reportable_exception &e)
2211 : : {
2212 [ + - ]: 58 : e.report(clog);
2213 : : // Report but swallow libc etc. errors here; let the caller
2214 : : // iterate to other matches of the content.
2215 : 58 : }
2216 : :
2217 : : return 0;
2218 : : }
2219 : :
2220 : :
2221 : : static int
2222 : 3235 : debuginfod_find_progress (debuginfod_client *, long a, long b)
2223 : : {
2224 [ - + ]: 3235 : if (verbose > 4)
2225 [ # # # # : 0 : obatched(clog) << "federated debuginfod progress=" << a << "/" << b << endl;
# # # # ]
2226 : :
2227 : 3235 : return interrupted;
2228 : : }
2229 : :
2230 : :
2231 : : // a little lru pool of debuginfod_client*s for reuse between query threads
2232 : :
2233 : : mutex dc_pool_lock;
2234 : : deque<debuginfod_client*> dc_pool;
2235 : :
2236 : 620 : debuginfod_client* debuginfod_pool_begin()
2237 : : {
2238 : 620 : unique_lock<mutex> lock(dc_pool_lock);
2239 [ + + ]: 620 : if (dc_pool.size() > 0)
2240 : : {
2241 [ + - + - : 1188 : inc_metric("dc_pool_op_count","op","begin-reuse");
+ - + - -
+ - + - -
- - ]
2242 : 594 : debuginfod_client *c = dc_pool.front();
2243 : 594 : dc_pool.pop_front();
2244 : 594 : return c;
2245 : : }
2246 [ + - + - : 52 : inc_metric("dc_pool_op_count","op","begin-new");
+ - + - -
+ - + - -
- - ]
2247 [ + - ]: 26 : return debuginfod_begin();
2248 : 620 : }
2249 : :
2250 : :
2251 : 143 : void debuginfod_pool_groom()
2252 : : {
2253 : 143 : unique_lock<mutex> lock(dc_pool_lock);
2254 [ + + ]: 169 : while (dc_pool.size() > 0)
2255 : : {
2256 [ + - + - : 52 : inc_metric("dc_pool_op_count","op","end");
+ - + - -
+ - + - -
- - ]
2257 [ + - ]: 26 : debuginfod_end(dc_pool.front());
2258 : 26 : dc_pool.pop_front();
2259 : : }
2260 : 143 : }
2261 : :
2262 : :
2263 : 620 : void debuginfod_pool_end(debuginfod_client* c)
2264 : : {
2265 : 620 : unique_lock<mutex> lock(dc_pool_lock);
2266 [ + - + - : 1240 : inc_metric("dc_pool_op_count","op","end-save");
+ - + - -
+ - + - -
- - ]
2267 [ + - ]: 620 : dc_pool.push_front(c); // accelerate reuse, vs. push_back
2268 : 620 : }
2269 : :
2270 : :
2271 : : static struct MHD_Response*
2272 : 1424 : handle_buildid (MHD_Connection* conn,
2273 : : const string& buildid /* unsafe */,
2274 : : string& artifacttype /* unsafe, cleanse on exception/return */,
2275 : : const string& suffix /* unsafe */,
2276 : : int *result_fd)
2277 : : {
2278 : : // validate artifacttype
2279 : 1424 : string atype_code;
2280 [ + + + - ]: 1424 : if (artifacttype == "debuginfo") atype_code = "D";
2281 [ + + + - ]: 686 : else if (artifacttype == "executable") atype_code = "E";
2282 [ + + + - ]: 68 : else if (artifacttype == "source") atype_code = "S";
2283 [ + + + - ]: 12 : else if (artifacttype == "section") atype_code = "I";
2284 : : else {
2285 [ + - ]: 4 : artifacttype = "invalid"; // PR28242 ensure http_resposes metrics don't propagate unclean user data
2286 [ + - + - ]: 12 : throw reportable_exception("invalid artifacttype");
2287 : : }
2288 : :
2289 [ + + ]: 1420 : if (conn != 0)
2290 [ + - + - : 3382 : inc_metric("http_requests_total", "type", artifacttype);
+ - - + -
- - + ]
2291 : :
2292 : 1420 : string section;
2293 [ + + ]: 1420 : if (atype_code == "I")
2294 : : {
2295 [ - + ]: 8 : if (suffix.size () < 2)
2296 [ # # # # ]: 0 : throw reportable_exception ("invalid section suffix");
2297 : :
2298 : : // Remove leading '/'
2299 [ + - - + ]: 8 : section = suffix.substr(1);
2300 : : }
2301 : :
2302 [ + + - + ]: 1476 : if (atype_code == "S" && suffix == "")
2303 [ # # # # ]: 0 : throw reportable_exception("invalid source suffix");
2304 : :
2305 : : // validate buildid
2306 [ + + ]: 1420 : if ((buildid.size() < 2) || // not empty
2307 [ + + + - : 2838 : (buildid.size() % 2) || // even number
+ - ]
2308 : 1418 : (buildid.find_first_not_of("0123456789abcdef") != string::npos)) // pure tasty lowercase hex
2309 [ + - - + ]: 4 : throw reportable_exception("invalid buildid");
2310 : :
2311 [ + - ]: 1418 : if (verbose > 1)
2312 [ + - + - ]: 4254 : obatched(clog) << "searching for buildid=" << buildid << " artifacttype=" << artifacttype
2313 [ + - + - : 1418 : << " suffix=" << suffix << endl;
+ - + - +
- ]
2314 : :
2315 : : // If invoked from the scanner threads, use the scanners' read-write
2316 : : // connection. Otherwise use the web query threads' read-only connection.
2317 [ + + ]: 1418 : sqlite3 *thisdb = (conn == 0) ? db : dbq;
2318 : :
2319 : 1418 : sqlite_ps *pp = 0;
2320 : :
2321 [ + + ]: 1418 : if (atype_code == "D")
2322 : : {
2323 [ - + ]: 738 : pp = new sqlite_ps (thisdb, "mhd-query-d",
2324 : : "select mtime, sourcetype, source0, source1 from " BUILDIDS "_query_d where buildid = ? "
2325 [ + - + - : 1476 : "order by mtime desc");
+ - + - +
- - - ]
2326 [ + - ]: 738 : pp->reset();
2327 [ + - ]: 738 : pp->bind(1, buildid);
2328 : : }
2329 [ + + ]: 680 : else if (atype_code == "E")
2330 : : {
2331 [ - + ]: 616 : pp = new sqlite_ps (thisdb, "mhd-query-e",
2332 : : "select mtime, sourcetype, source0, source1 from " BUILDIDS "_query_e where buildid = ? "
2333 [ + - + - : 1232 : "order by mtime desc");
+ - + - +
- - - ]
2334 [ + - ]: 616 : pp->reset();
2335 [ + - ]: 616 : pp->bind(1, buildid);
2336 : : }
2337 [ + + ]: 64 : else if (atype_code == "S")
2338 : : {
2339 : : // PR25548
2340 : : // Incoming source queries may come in with either dwarf-level OR canonicalized paths.
2341 : : // We let the query pass with either one.
2342 : :
2343 [ - + ]: 56 : pp = new sqlite_ps (thisdb, "mhd-query-s",
2344 : : "select mtime, sourcetype, source0, source1 from " BUILDIDS "_query_s where buildid = ? and artifactsrc in (?,?) "
2345 [ + - + - : 112 : "order by sharedprefix(source0,source0ref) desc, mtime desc");
+ - + - +
- - - ]
2346 [ + - ]: 56 : pp->reset();
2347 [ + - ]: 56 : pp->bind(1, buildid);
2348 : : // NB: we don't store the non-canonicalized path names any more, but old databases
2349 : : // might have them (and no canon ones), so we keep searching for both.
2350 [ + - ]: 56 : pp->bind(2, suffix);
2351 [ + - + - : 730 : pp->bind(3, canon_pathname(suffix));
- + ]
2352 : : }
2353 [ + - ]: 8 : else if (atype_code == "I")
2354 : : {
2355 [ - + ]: 8 : pp = new sqlite_ps (thisdb, "mhd-query-i",
2356 : : "select mtime, sourcetype, source0, source1, 1 as debug_p from " BUILDIDS "_query_d where buildid = ? "
2357 : : "union all "
2358 : : "select mtime, sourcetype, source0, source1, 0 as debug_p from " BUILDIDS "_query_e where buildid = ? "
2359 [ + - + - : 16 : "order by debug_p desc, mtime desc");
+ - + - +
- - - ]
2360 [ + - ]: 8 : pp->reset();
2361 [ + - ]: 8 : pp->bind(1, buildid);
2362 [ + - ]: 8 : pp->bind(2, buildid);
2363 : : }
2364 : 1418 : unique_ptr<sqlite_ps> ps_closer(pp); // release pp if exception or return
2365 : :
2366 : 1418 : bool do_upstream_section_query = true;
2367 : :
2368 : : // consume all the rows
2369 : 1480 : while (1)
2370 : : {
2371 [ + - ]: 1480 : int rc = pp->step();
2372 [ + + ]: 1480 : if (rc == SQLITE_DONE) break;
2373 [ - + ]: 860 : if (rc != SQLITE_ROW)
2374 [ # # # # ]: 0 : throw sqlite_exception(rc, "step");
2375 : :
2376 [ + - ]: 860 : int64_t b_mtime = sqlite3_column_int64 (*pp, 0);
2377 [ + - - + : 860 : string b_stype = string((const char*) sqlite3_column_text (*pp, 1) ?: ""); /* by DDL may not be NULL */
+ - ]
2378 [ + - - + : 860 : string b_source0 = string((const char*) sqlite3_column_text (*pp, 2) ?: ""); /* may be NULL */
+ - - - ]
2379 [ + - + + : 934 : string b_source1 = string((const char*) sqlite3_column_text (*pp, 3) ?: ""); /* may be NULL */
+ - - - ]
2380 : :
2381 [ + - ]: 860 : if (verbose > 1)
2382 [ + - + - : 2580 : obatched(clog) << "found mtime=" << b_mtime << " stype=" << b_stype
- - ]
2383 [ + - + - : 860 : << " source0=" << b_source0 << " source1=" << b_source1 << endl;
+ - + - +
- + - +
- ]
2384 : :
2385 : : // Try accessing the located match.
2386 : : // XXX: in case of multiple matches, attempt them in parallel?
2387 [ + - ]: 860 : auto r = handle_buildid_match (conn ? false : true,
2388 : : b_mtime, b_stype, b_source0, b_source1,
2389 : : section, result_fd);
2390 [ + + ]: 860 : if (r)
2391 [ + + ]: 798 : return r;
2392 : :
2393 : : // If a debuginfo file matching BUILDID was found but didn't contain
2394 : : // the desired section, then the section should not exist. Don't
2395 : : // bother querying upstream servers.
2396 [ + + + - : 62 : if (!section.empty () && (sqlite3_column_int (*pp, 4) == 1))
- + ]
2397 : : {
2398 : 4 : struct stat st;
2399 : :
2400 : : // For "F" sourcetype, check if the debuginfo exists. For "R"
2401 : : // sourcetype, check if the debuginfo was interned into the fdcache.
2402 [ - + ]: 6 : if ((b_stype == "F" && (stat (b_source0.c_str (), &st) == 0))
2403 [ + + + - : 6 : || (b_stype == "R" && fdcache.probe (b_source0, b_source1)))
+ - - + ]
2404 : : do_upstream_section_query = false;
2405 : : }
2406 [ + - - + : 1720 : }
+ - - + ]
2407 [ + - ]: 620 : pp->reset();
2408 : :
2409 [ - + ]: 620 : if (!do_upstream_section_query)
2410 [ # # # # ]: 0 : throw reportable_exception(MHD_HTTP_NOT_FOUND, "not found");
2411 : :
2412 : : // We couldn't find it in the database. Last ditch effort
2413 : : // is to defer to other debuginfo servers.
2414 : :
2415 : 620 : int fd = -1;
2416 [ + - ]: 620 : debuginfod_client *client = debuginfod_pool_begin ();
2417 [ - + ]: 620 : if (client == NULL)
2418 [ # # # # ]: 0 : throw libc_exception(errno, "debuginfod client pool alloc");
2419 : 620 : defer_dtor<debuginfod_client*,void> client_closer (client, debuginfod_pool_end);
2420 : :
2421 [ + - ]: 620 : debuginfod_set_progressfn (client, & debuginfod_find_progress);
2422 : :
2423 [ + - ]: 620 : if (conn)
2424 : : {
2425 : : // Transcribe incoming User-Agent:
2426 [ + - - + : 620 : string ua = MHD_lookup_connection_value (conn, MHD_HEADER_KIND, "User-Agent") ?: "";
+ - ]
2427 [ + - + - : 624 : string ua_complete = string("User-Agent: ") + ua;
+ - ]
2428 [ + - ]: 620 : debuginfod_add_http_header (client, ua_complete.c_str());
2429 : :
2430 : : // Compute larger XFF:, for avoiding info loss during
2431 : : // federation, and for future cyclicity detection.
2432 [ + - + + : 1226 : string xff = MHD_lookup_connection_value (conn, MHD_HEADER_KIND, "X-Forwarded-For") ?: "";
+ - + - ]
2433 [ + + ]: 620 : if (xff != "")
2434 [ + - - + ]: 36 : xff += string(", "); // comma separated list
2435 : :
2436 : 620 : unsigned int xff_count = 0;
2437 [ + + ]: 860 : for (auto&& i : xff){
2438 [ + + ]: 240 : if (i == ',') xff_count++;
2439 : : }
2440 : :
2441 : : // if X-Forwarded-For: exceeds N hops,
2442 : : // do not delegate a local lookup miss to upstream debuginfods.
2443 [ + + ]: 620 : if (xff_count >= forwarded_ttl_limit)
2444 [ + - + - ]: 8 : throw reportable_exception(MHD_HTTP_NOT_FOUND, "not found, --forwared-ttl-limit reached \
2445 : 8 : and will not query the upstream servers");
2446 : :
2447 : : // Compute the client's numeric IP address only - so can't merge with conninfo()
2448 [ + - ]: 616 : const union MHD_ConnectionInfo *u = MHD_get_connection_info (conn,
2449 : : MHD_CONNECTION_INFO_CLIENT_ADDRESS);
2450 [ + - ]: 616 : struct sockaddr *so = u ? u->client_addr : 0;
2451 : 616 : char hostname[256] = ""; // RFC1035
2452 [ + - - + ]: 616 : if (so && so->sa_family == AF_INET) {
2453 [ # # ]: 0 : (void) getnameinfo (so, sizeof (struct sockaddr_in), hostname, sizeof (hostname), NULL, 0,
2454 : : NI_NUMERICHOST);
2455 [ + - ]: 616 : } else if (so && so->sa_family == AF_INET6) {
2456 : 616 : struct sockaddr_in6* addr6 = (struct sockaddr_in6*) so;
2457 [ + - + - : 616 : if (IN6_IS_ADDR_V4MAPPED(&addr6->sin6_addr)) {
- + ]
2458 : 616 : struct sockaddr_in addr4;
2459 [ + - ]: 616 : memset (&addr4, 0, sizeof(addr4));
2460 : 616 : addr4.sin_family = AF_INET;
2461 : 616 : addr4.sin_port = addr6->sin6_port;
2462 [ + - ]: 616 : memcpy (&addr4.sin_addr.s_addr, addr6->sin6_addr.s6_addr+12, sizeof(addr4.sin_addr.s_addr));
2463 [ + - ]: 616 : (void) getnameinfo ((struct sockaddr*) &addr4, sizeof (addr4),
2464 : : hostname, sizeof (hostname), NULL, 0,
2465 : : NI_NUMERICHOST);
2466 : : } else {
2467 [ # # ]: 0 : (void) getnameinfo (so, sizeof (struct sockaddr_in6), hostname, sizeof (hostname), NULL, 0,
2468 : : NI_NUMERICHOST);
2469 : : }
2470 : : }
2471 : :
2472 [ + - + - : 1236 : string xff_complete = string("X-Forwarded-For: ")+xff+string(hostname);
+ - + - -
+ - + - -
- + ]
2473 [ + - ]: 616 : debuginfod_add_http_header (client, xff_complete.c_str());
2474 [ + + + - : 1300 : }
+ + ]
2475 : :
2476 [ + + ]: 616 : if (artifacttype == "debuginfo")
2477 [ + - ]: 68 : fd = debuginfod_find_debuginfo (client,
2478 [ + - ]: 68 : (const unsigned char*) buildid.c_str(),
2479 : : 0, NULL);
2480 [ + + ]: 548 : else if (artifacttype == "executable")
2481 [ + - ]: 546 : fd = debuginfod_find_executable (client,
2482 [ + - ]: 546 : (const unsigned char*) buildid.c_str(),
2483 : : 0, NULL);
2484 [ + - ]: 2 : else if (artifacttype == "source")
2485 [ + - ]: 2 : fd = debuginfod_find_source (client,
2486 [ + - ]: 2 : (const unsigned char*) buildid.c_str(),
2487 : : 0, suffix.c_str(), NULL);
2488 [ # # ]: 0 : else if (artifacttype == "section")
2489 [ # # ]: 0 : fd = debuginfod_find_section (client,
2490 [ # # ]: 0 : (const unsigned char*) buildid.c_str(),
2491 : : 0, section.c_str(), NULL);
2492 : :
2493 [ + + ]: 616 : if (fd >= 0)
2494 : : {
2495 [ + - ]: 4 : if (conn != 0)
2496 [ + - + - : 624 : inc_metric ("http_responses_total","result","upstream");
+ - + - -
+ - + - -
- - ]
2497 : 4 : struct stat s;
2498 : 4 : int rc = fstat (fd, &s);
2499 [ + - ]: 4 : if (rc == 0)
2500 : : {
2501 [ + - ]: 4 : auto r = MHD_create_response_from_fd ((uint64_t) s.st_size, fd);
2502 [ + - ]: 4 : if (r)
2503 : : {
2504 [ + - ]: 4 : add_mhd_response_header (r, "Content-Type",
2505 : : "application/octet-stream");
2506 : : // Copy the incoming headers
2507 [ + - ]: 4 : const char * hdrs = debuginfod_get_headers(client);
2508 [ + - ]: 4 : string header_dup;
2509 [ + - ]: 4 : if (hdrs)
2510 [ + - - + ]: 4 : header_dup = string(hdrs);
2511 : : // Parse the "header: value\n" lines into (h,v) tuples and pass on
2512 : 20 : while(1)
2513 : : {
2514 : 12 : size_t newline = header_dup.find('\n');
2515 [ + + ]: 12 : if (newline == string::npos) break;
2516 : 8 : size_t colon = header_dup.find(':');
2517 [ + - ]: 8 : if (colon == string::npos) break;
2518 [ + - ]: 8 : string header = header_dup.substr(0,colon);
2519 [ + - ]: 8 : string value = header_dup.substr(colon+1,newline-colon-1);
2520 : : // strip leading spaces from value
2521 : 8 : size_t nonspace = value.find_first_not_of(" ");
2522 [ + - ]: 8 : if (nonspace != string::npos)
2523 [ + - - + ]: 8 : value = value.substr(nonspace);
2524 [ + - ]: 8 : add_mhd_response_header(r, header.c_str(), value.c_str());
2525 [ + - + + : 12 : header_dup = header_dup.substr(newline+1);
- + - - ]
2526 [ + - ]: 16 : }
2527 : :
2528 [ + - ]: 4 : add_mhd_last_modified (r, s.st_mtime);
2529 [ + - ]: 4 : if (verbose > 1)
2530 [ + - + - : 8 : obatched(clog) << "serving file from upstream debuginfod/cache" << endl;
- - ]
2531 [ + - ]: 4 : if (result_fd)
2532 : 4 : *result_fd = fd;
2533 [ + - ]: 4 : return r; // NB: don't close fd; libmicrohttpd will
2534 : 4 : }
2535 : : }
2536 [ # # ]: 0 : close (fd);
2537 : : }
2538 : : else
2539 [ + + ]: 612 : switch(fd)
2540 : : {
2541 : : case -ENOSYS:
2542 : : break;
2543 : : case -ENOENT:
2544 : : break;
2545 : 532 : default: // some more tricky error
2546 [ + - + - ]: 1064 : throw libc_exception(-fd, "upstream debuginfod query failed");
2547 : : }
2548 : :
2549 [ + - - + ]: 160 : throw reportable_exception(MHD_HTTP_NOT_FOUND, "not found");
2550 [ - + - + ]: 2036 : }
2551 : :
2552 : :
2553 : : ////////////////////////////////////////////////////////////////////////
2554 : :
2555 : : static map<string,double> metrics; // arbitrary data for /metrics query
2556 : : // NB: store int64_t since all our metrics are integers; prometheus accepts double
2557 : : static mutex metrics_lock;
2558 : : // NB: these objects get released during the process exit via global dtors
2559 : : // do not call them from within other global dtors
2560 : :
2561 : : // utility function for assembling prometheus-compatible
2562 : : // name="escaped-value" strings
2563 : : // https://prometheus.io/docs/instrumenting/exposition_formats/
2564 : : static string
2565 : 256126 : metric_label(const string& name, const string& value)
2566 : : {
2567 : 256126 : string x = name + "=\"";
2568 [ + + ]: 4117220 : for (auto&& c : value)
2569 [ - - - + ]: 3861211 : switch(c)
2570 : : {
2571 [ # # ]: 0 : case '\\': x += "\\\\"; break;
2572 [ # # ]: 0 : case '\"': x += "\\\""; break;
2573 [ # # ]: 0 : case '\n': x += "\\n"; break;
2574 [ + - ]: 7722347 : default: x += c; break;
2575 : : }
2576 [ + - ]: 256009 : x += "\"";
2577 : 256058 : return x;
2578 : 0 : }
2579 : :
2580 : :
2581 : : // add prometheus-format metric name + label tuple (if any) + value
2582 : :
2583 : : static void
2584 : 8816 : set_metric(const string& metric, double value)
2585 : : {
2586 : 8816 : unique_lock<mutex> lock(metrics_lock);
2587 [ + - ]: 8816 : metrics[metric] = value;
2588 : 8816 : }
2589 : : #if 0 /* unused */
2590 : : static void
2591 : : inc_metric(const string& metric)
2592 : : {
2593 : : unique_lock<mutex> lock(metrics_lock);
2594 : : metrics[metric] ++;
2595 : : }
2596 : : #endif
2597 : : static void
2598 : 7152 : set_metric(const string& metric,
2599 : : const string& lname, const string& lvalue,
2600 : : double value)
2601 : : {
2602 [ + - + - : 14305 : string key = (metric + "{" + metric_label(lname, lvalue) + "}");
- + - + +
+ - - ]
2603 [ + - ]: 7153 : unique_lock<mutex> lock(metrics_lock);
2604 [ + - ]: 7153 : metrics[key] = value;
2605 [ + - ]: 14306 : }
2606 : :
2607 : : static void
2608 : 114050 : inc_metric(const string& metric,
2609 : : const string& lname, const string& lvalue)
2610 : : {
2611 [ + - + - : 245497 : string key = (metric + "{" + metric_label(lname, lvalue) + "}");
- + + + +
+ - - ]
2612 [ + - ]: 114050 : unique_lock<mutex> lock(metrics_lock);
2613 [ + - ]: 114059 : metrics[key] ++;
2614 [ + - ]: 228106 : }
2615 : : static void
2616 : 110111 : add_metric(const string& metric,
2617 : : const string& lname, const string& lvalue,
2618 : : double value)
2619 : : {
2620 [ + - + - : 237625 : string key = (metric + "{" + metric_label(lname, lvalue) + "}");
- + + + +
+ - - ]
2621 [ + - ]: 110119 : unique_lock<mutex> lock(metrics_lock);
2622 [ + - ]: 110135 : metrics[key] += value;
2623 [ + - ]: 220268 : }
2624 : : #if 0
2625 : : static void
2626 : : add_metric(const string& metric,
2627 : : double value)
2628 : : {
2629 : : unique_lock<mutex> lock(metrics_lock);
2630 : : metrics[metric] += value;
2631 : : }
2632 : : #endif
2633 : :
2634 : :
2635 : : // and more for higher arity labels if needed
2636 : :
2637 : : static void
2638 : 6228 : inc_metric(const string& metric,
2639 : : const string& lname, const string& lvalue,
2640 : : const string& rname, const string& rvalue)
2641 : : {
2642 [ + - - + : 12456 : string key = (metric + "{"
- - ]
2643 [ + - + - : 24912 : + metric_label(lname, lvalue) + ","
- + - + +
+ - - ]
2644 [ + - - + : 18684 : + metric_label(rname, rvalue) + "}");
- + ]
2645 [ + - ]: 6228 : unique_lock<mutex> lock(metrics_lock);
2646 [ + - ]: 6228 : metrics[key] ++;
2647 [ + - ]: 12456 : }
2648 : : static void
2649 : 6228 : add_metric(const string& metric,
2650 : : const string& lname, const string& lvalue,
2651 : : const string& rname, const string& rvalue,
2652 : : double value)
2653 : : {
2654 [ + - - + : 12456 : string key = (metric + "{"
- - ]
2655 [ + - + - : 24912 : + metric_label(lname, lvalue) + ","
- + - + +
+ - - ]
2656 [ + - - + : 18684 : + metric_label(rname, rvalue) + "}");
- + ]
2657 [ + - ]: 6228 : unique_lock<mutex> lock(metrics_lock);
2658 [ + - ]: 6228 : metrics[key] += value;
2659 [ + - ]: 12456 : }
2660 : :
2661 : : static struct MHD_Response*
2662 : 686 : handle_metrics (off_t* size)
2663 : : {
2664 : 686 : stringstream o;
2665 : 686 : {
2666 [ + - ]: 686 : unique_lock<mutex> lock(metrics_lock);
2667 [ + + ]: 64435 : for (auto&& i : metrics)
2668 [ + - ]: 63749 : o << i.first
2669 : : << " "
2670 [ + - + - ]: 63749 : << std::setprecision(std::numeric_limits<double>::digits10 + 1)
2671 [ + - + - ]: 63749 : << i.second
2672 : 63749 : << endl;
2673 : 0 : }
2674 [ + - ]: 686 : const string& os = o.str();
2675 [ + - ]: 686 : MHD_Response* r = MHD_create_response_from_buffer (os.size(),
2676 [ + - ]: 686 : (void*) os.c_str(),
2677 : : MHD_RESPMEM_MUST_COPY);
2678 [ + - ]: 686 : if (r != NULL)
2679 : : {
2680 [ + - ]: 686 : *size = os.size();
2681 [ + - ]: 686 : add_mhd_response_header (r, "Content-Type", "text/plain");
2682 : : }
2683 [ + - ]: 1372 : return r;
2684 : 686 : }
2685 : :
2686 : : static struct MHD_Response*
2687 : 0 : handle_root (off_t* size)
2688 : : {
2689 [ # # # # : 0 : static string version = "debuginfod (" + string (PACKAGE_NAME) + ") "
# # # # #
# # # ]
2690 [ # # # # : 0 : + string (PACKAGE_VERSION);
# # # # #
# ]
2691 : 0 : MHD_Response* r = MHD_create_response_from_buffer (version.size (),
2692 : 0 : (void *) version.c_str (),
2693 : : MHD_RESPMEM_PERSISTENT);
2694 [ # # ]: 0 : if (r != NULL)
2695 : : {
2696 : 0 : *size = version.size ();
2697 : 0 : add_mhd_response_header (r, "Content-Type", "text/plain");
2698 : : }
2699 : 0 : return r;
2700 : : }
2701 : :
2702 : :
2703 : : ////////////////////////////////////////////////////////////////////////
2704 : :
2705 : :
2706 : : /* libmicrohttpd callback */
2707 : : static MHD_RESULT
2708 : 4152 : handler_cb (void * /*cls*/,
2709 : : struct MHD_Connection *connection,
2710 : : const char *url,
2711 : : const char *method,
2712 : : const char * /*version*/,
2713 : : const char * /*upload_data*/,
2714 : : size_t * /*upload_data_size*/,
2715 : : void ** ptr)
2716 : : {
2717 : 4152 : struct MHD_Response *r = NULL;
2718 : 4152 : string url_copy = url;
2719 : :
2720 : : /* libmicrohttpd always makes (at least) two callbacks: once just
2721 : : past the headers, and one after the request body is finished
2722 : : being received. If we process things early (first callback) and
2723 : : queue a response, libmicrohttpd would suppress http keep-alive
2724 : : (via connection->read_closed = true). */
2725 : 4152 : static int aptr; /* just some random object to use as a flag */
2726 [ + + ]: 4152 : if (&aptr != *ptr)
2727 : : {
2728 : : /* do never respond on first call */
2729 : 2076 : *ptr = &aptr;
2730 : 2076 : return MHD_YES;
2731 : : }
2732 : 2076 : *ptr = NULL; /* reset when done */
2733 : :
2734 [ + - ]: 2076 : const char *maxsize_string = MHD_lookup_connection_value(connection, MHD_HEADER_KIND, "X-DEBUGINFOD-MAXSIZE");
2735 : 2076 : long maxsize = 0;
2736 [ + + + - ]: 2076 : if (maxsize_string != NULL && maxsize_string[0] != '\0')
2737 : 2 : maxsize = atol(maxsize_string);
2738 : : else
2739 : : maxsize = 0;
2740 : :
2741 : : #if MHD_VERSION >= 0x00097002
2742 : 2076 : enum MHD_Result rc;
2743 : : #else
2744 : : int rc = MHD_NO; // mhd
2745 : : #endif
2746 : 2076 : int http_code = 500;
2747 : 2076 : off_t http_size = -1;
2748 : 2076 : struct timespec ts_start, ts_end;
2749 : 2076 : clock_gettime (CLOCK_MONOTONIC, &ts_start);
2750 : 2076 : double afteryou = 0.0;
2751 [ + - ]: 2076 : string artifacttype, suffix;
2752 : :
2753 : 2076 : try
2754 : : {
2755 [ + - - + : 4782 : if (string(method) != "GET")
- + ]
2756 [ # # # # ]: 0 : throw reportable_exception(400, "we support GET only");
2757 : :
2758 : : /* Start decoding the URL. */
2759 : 2076 : size_t slash1 = url_copy.find('/', 1);
2760 [ + - ]: 2076 : string url1 = url_copy.substr(0, slash1); // ok even if slash1 not found
2761 : :
2762 [ + + + - ]: 3460 : if (slash1 != string::npos && url1 == "/buildid")
2763 : : {
2764 : : // PR27863: block this thread awhile if another thread is already busy
2765 : : // fetching the exact same thing. This is better for Everyone.
2766 : : // The latecomer says "... after you!" and waits.
2767 [ + - + - : 3390 : add_metric ("thread_busy", "role", "http-buildid-after-you", 1);
+ - + - -
+ + - - -
- - ]
2768 : : #ifdef HAVE_PTHREAD_SETNAME_NP
2769 : 1384 : (void) pthread_setname_np (pthread_self(), "mhd-buildid-after-you");
2770 : : #endif
2771 : 1384 : struct timespec tsay_start, tsay_end;
2772 : 1384 : clock_gettime (CLOCK_MONOTONIC, &tsay_start);
2773 [ + + + - ]: 1384 : static unique_set<string> busy_urls;
2774 [ + - ]: 1384 : unique_set_reserver<string> after_you(busy_urls, url_copy);
2775 : 1384 : clock_gettime (CLOCK_MONOTONIC, &tsay_end);
2776 : 1384 : afteryou = (tsay_end.tv_sec - tsay_start.tv_sec) + (tsay_end.tv_nsec - tsay_start.tv_nsec)/1.e9;
2777 [ + - + - : 2768 : add_metric ("thread_busy", "role", "http-buildid-after-you", -1);
+ - + - -
+ + - - -
- - ]
2778 : :
2779 [ + - + - : 2768 : tmp_inc_metric m ("thread_busy", "role", "http-buildid");
+ - + - -
+ - + - -
- - ]
2780 : : #ifdef HAVE_PTHREAD_SETNAME_NP
2781 : 1384 : (void) pthread_setname_np (pthread_self(), "mhd-buildid");
2782 : : #endif
2783 : 1384 : size_t slash2 = url_copy.find('/', slash1+1);
2784 [ - + ]: 1384 : if (slash2 == string::npos)
2785 [ # # # # ]: 0 : throw reportable_exception("/buildid/ webapi error, need buildid");
2786 : :
2787 [ + - ]: 1384 : string buildid = url_copy.substr(slash1+1, slash2-slash1-1);
2788 : :
2789 : 1384 : size_t slash3 = url_copy.find('/', slash2+1);
2790 : :
2791 [ + + ]: 1384 : if (slash3 == string::npos)
2792 : : {
2793 [ + - - + ]: 1320 : artifacttype = url_copy.substr(slash2+1);
2794 [ + - ]: 1320 : suffix = "";
2795 : : }
2796 : : else
2797 : : {
2798 [ + - - + ]: 64 : artifacttype = url_copy.substr(slash2+1, slash3-slash2-1);
2799 [ + - - + : 686 : suffix = url_copy.substr(slash3); // include the slash in the suffix
+ + ]
2800 : : }
2801 : :
2802 : : // get the resulting fd so we can report its size
2803 : 1384 : int fd;
2804 [ + + ]: 1384 : r = handle_buildid(connection, buildid, artifacttype, suffix, &fd);
2805 [ + - ]: 762 : if (r)
2806 : : {
2807 : 762 : struct stat fs;
2808 [ + - ]: 762 : if (fstat(fd, &fs) == 0)
2809 : 762 : http_size = fs.st_size;
2810 : : // libmicrohttpd will close (fd);
2811 : : }
2812 : 2006 : }
2813 [ + + ]: 692 : else if (url1 == "/metrics")
2814 : : {
2815 [ + - + - : 1372 : tmp_inc_metric m ("thread_busy", "role", "http-metrics");
+ - + - -
+ - + - -
- - ]
2816 [ + - ]: 686 : artifacttype = "metrics";
2817 [ + - + - : 1372 : inc_metric("http_requests_total", "type", artifacttype);
+ - - + -
- ]
2818 [ + - ]: 686 : r = handle_metrics(& http_size);
2819 : 686 : }
2820 [ - + ]: 6 : else if (url1 == "/")
2821 : : {
2822 [ # # ]: 0 : artifacttype = "/";
2823 [ - - - - : 630 : inc_metric("http_requests_total", "type", artifacttype);
- - - - -
- - + ]
2824 [ # # ]: 0 : r = handle_root(& http_size);
2825 : : }
2826 : : else
2827 [ + - + - : 18 : throw reportable_exception("webapi error, unrecognized '" + url1 + "'");
+ - - + ]
2828 : :
2829 [ - + ]: 1448 : if (r == 0)
2830 [ # # # # ]: 0 : throw reportable_exception("internal error, missing response");
2831 : :
2832 [ + + + - ]: 1448 : if (maxsize > 0 && http_size > maxsize)
2833 : : {
2834 [ + - ]: 2 : MHD_destroy_response(r);
2835 [ + - + - : 6 : throw reportable_exception(406, "File too large, max size=" + std::to_string(maxsize));
+ - - + ]
2836 : : }
2837 : :
2838 [ + - ]: 1446 : rc = MHD_queue_response (connection, MHD_HTTP_OK, r);
2839 : 1446 : http_code = MHD_HTTP_OK;
2840 [ + - ]: 1446 : MHD_destroy_response (r);
2841 : 2076 : }
2842 [ - + ]: 630 : catch (const reportable_exception& e)
2843 : : {
2844 [ + - + - : 1260 : inc_metric("http_responses_total","result","error");
+ - + - -
+ - + - -
- - ]
2845 [ + - ]: 630 : e.report(clog);
2846 : 630 : http_code = e.code;
2847 [ + - ]: 630 : http_size = e.message.size();
2848 [ + - ]: 630 : rc = e.mhd_send_response (connection);
2849 : 630 : }
2850 : :
2851 : 2076 : clock_gettime (CLOCK_MONOTONIC, &ts_end);
2852 : 2076 : double deltas = (ts_end.tv_sec - ts_start.tv_sec) + (ts_end.tv_nsec - ts_start.tv_nsec)/1.e9;
2853 : : // afteryou: delay waiting for other client's identical query to complete
2854 : : // deltas: total latency, including afteryou waiting
2855 [ + - + - : 4152 : obatched(clog) << conninfo(connection)
- - ]
2856 : : << ' ' << method << ' ' << url
2857 [ + - + - : 2076 : << ' ' << http_code << ' ' << http_size
+ - + - +
- + - +
- ]
2858 [ + - + - : 2076 : << ' ' << (int)(afteryou*1000) << '+' << (int)((deltas-afteryou)*1000) << "ms"
+ - + - +
- + - +
- ]
2859 [ + - ]: 2076 : << endl;
2860 : :
2861 : : // related prometheus metrics
2862 : 2076 : string http_code_str = to_string(http_code);
2863 [ + - + - : 4152 : add_metric("http_responses_transfer_bytes_sum",
+ - + - -
+ - + - -
- - ]
2864 : : "code", http_code_str, "type", artifacttype, http_size);
2865 [ + - + - : 4152 : inc_metric("http_responses_transfer_bytes_count",
+ - + - -
+ - + - -
- - ]
2866 : : "code", http_code_str, "type", artifacttype);
2867 : :
2868 [ + - + - : 4152 : add_metric("http_responses_duration_milliseconds_sum",
+ - + - -
+ - + - -
- - ]
2869 : : "code", http_code_str, "type", artifacttype, deltas*1000); // prometheus prefers _seconds and floating point
2870 [ + - + - : 4152 : inc_metric("http_responses_duration_milliseconds_count",
+ - + - -
+ - + - -
- - ]
2871 : : "code", http_code_str, "type", artifacttype);
2872 : :
2873 [ + - + - : 4152 : add_metric("http_responses_after_you_milliseconds_sum",
+ - + - -
+ - + - -
- - ]
2874 : : "code", http_code_str, "type", artifacttype, afteryou*1000);
2875 [ + - + - : 4152 : inc_metric("http_responses_after_you_milliseconds_count",
+ - + - -
+ - + - -
- - - - ]
2876 : : "code", http_code_str, "type", artifacttype);
2877 : :
2878 [ - + ]: 2076 : return rc;
2879 [ + + - + : 9052 : }
+ + ]
2880 : :
2881 : :
2882 : : ////////////////////////////////////////////////////////////////////////
2883 : : // borrowed originally from src/nm.c get_local_names()
2884 : :
2885 : : static void
2886 : 210 : dwarf_extract_source_paths (Elf *elf, set<string>& debug_sourcefiles)
2887 : : noexcept // no exceptions - so we can simplify the altdbg resource release at end
2888 : : {
2889 : 210 : Dwarf* dbg = dwarf_begin_elf (elf, DWARF_C_READ, NULL);
2890 [ - + ]: 210 : if (dbg == NULL)
2891 : 0 : return;
2892 : :
2893 : 210 : Dwarf* altdbg = NULL;
2894 : 210 : int altdbg_fd = -1;
2895 : :
2896 : : // DWZ handling: if we have an unsatisfied debug-alt-link, add an
2897 : : // empty string into the outgoing sourcefiles set, so the caller
2898 : : // should know that our data is incomplete.
2899 : 210 : const char *alt_name_p;
2900 : 210 : const void *alt_build_id; // elfutils-owned memory
2901 : 210 : ssize_t sz = dwelf_dwarf_gnu_debugaltlink (dbg, &alt_name_p, &alt_build_id);
2902 [ + + ]: 210 : if (sz > 0) // got one!
2903 : : {
2904 : 40 : string buildid;
2905 : 40 : unsigned char* build_id_bytes = (unsigned char*) alt_build_id;
2906 [ + + ]: 840 : for (ssize_t idx=0; idx<sz; idx++)
2907 : : {
2908 : 800 : buildid += "0123456789abcdef"[build_id_bytes[idx] >> 4];
2909 : 800 : buildid += "0123456789abcdef"[build_id_bytes[idx] & 0xf];
2910 : : }
2911 : :
2912 [ + - ]: 40 : if (verbose > 3)
2913 : 40 : obatched(clog) << "Need altdebug buildid=" << buildid << endl;
2914 : :
2915 : : // but is it unsatisfied the normal elfutils ways?
2916 : 40 : Dwarf* alt = dwarf_getalt (dbg);
2917 [ + - ]: 40 : if (alt == NULL)
2918 : : {
2919 : : // Yup, unsatisfied the normal way. Maybe we can satisfy it
2920 : : // from our own debuginfod database.
2921 : 40 : int alt_fd;
2922 : 40 : struct MHD_Response *r = 0;
2923 : 40 : try
2924 : : {
2925 [ + - ]: 40 : string artifacttype = "debuginfo";
2926 [ + - + - : 40 : r = handle_buildid (0, buildid, artifacttype, "", &alt_fd);
- + - + -
- ]
2927 : 0 : }
2928 [ - - ]: 0 : catch (const reportable_exception& e)
2929 : : {
2930 : : // swallow exceptions
2931 : 0 : }
2932 : :
2933 : : // NB: this is not actually recursive! This invokes the web-query
2934 : : // path, which cannot get back into the scan code paths.
2935 [ + - ]: 40 : if (r)
2936 : : {
2937 : : // Found it!
2938 : 40 : altdbg_fd = dup(alt_fd); // ok if this fails, downstream failures ok
2939 : 40 : alt = altdbg = dwarf_begin (altdbg_fd, DWARF_C_READ);
2940 : : // NB: must close this dwarf and this fd at the bottom of the function!
2941 : 40 : MHD_destroy_response (r); // will close alt_fd
2942 [ + - ]: 40 : if (alt)
2943 : 40 : dwarf_setalt (dbg, alt);
2944 : : }
2945 : : }
2946 : : else
2947 : : {
2948 : : // NB: dwarf_setalt(alt) inappropriate - already done!
2949 : : // NB: altdbg will stay 0 so nothing tries to redundantly dealloc.
2950 : : }
2951 : :
2952 [ + - ]: 40 : if (alt)
2953 : : {
2954 [ + - ]: 40 : if (verbose > 3)
2955 : 40 : obatched(clog) << "Resolved altdebug buildid=" << buildid << endl;
2956 : : }
2957 : : else // (alt == NULL) - signal possible presence of poor debuginfo
2958 : : {
2959 [ # # ]: 0 : debug_sourcefiles.insert("");
2960 [ # # ]: 0 : if (verbose > 3)
2961 : 0 : obatched(clog) << "Unresolved altdebug buildid=" << buildid << endl;
2962 : : }
2963 : 40 : }
2964 : :
2965 : 210 : Dwarf_Off offset = 0;
2966 : 210 : Dwarf_Off old_offset;
2967 : 210 : size_t hsize;
2968 : :
2969 [ + + ]: 2310 : while (dwarf_nextcu (dbg, old_offset = offset, &offset, &hsize, NULL, NULL, NULL) == 0)
2970 : : {
2971 : 2100 : Dwarf_Die cudie_mem;
2972 : 2100 : Dwarf_Die *cudie = dwarf_offdie (dbg, old_offset + hsize, &cudie_mem);
2973 : :
2974 [ - + ]: 2100 : if (cudie == NULL)
2975 : 20 : continue;
2976 [ + + ]: 2100 : if (dwarf_tag (cudie) != DW_TAG_compile_unit)
2977 : 20 : continue;
2978 : :
2979 [ - + ]: 2080 : const char *cuname = dwarf_diename(cudie) ?: "unknown";
2980 : :
2981 : 2080 : Dwarf_Files *files;
2982 : 2080 : size_t nfiles;
2983 [ - + ]: 2080 : if (dwarf_getsrcfiles (cudie, &files, &nfiles) != 0)
2984 : 0 : continue;
2985 : :
2986 : : // extract DW_AT_comp_dir to resolve relative file names
2987 : 2080 : const char *comp_dir = "";
2988 : 2080 : const char *const *dirs;
2989 : 2080 : size_t ndirs;
2990 [ - + ]: 2080 : if (dwarf_getsrcdirs (files, &dirs, &ndirs) == 0 &&
2991 [ - + ]: 2080 : dirs[0] != NULL)
2992 : : comp_dir = dirs[0];
2993 : : if (comp_dir == NULL)
2994 : : comp_dir = "";
2995 : :
2996 [ + + ]: 2080 : if (verbose > 3)
2997 : 328 : obatched(clog) << "searching for sources for cu=" << cuname << " comp_dir=" << comp_dir
2998 : 164 : << " #files=" << nfiles << " #dirs=" << ndirs << endl;
2999 : :
3000 [ + - - - ]: 2080 : if (comp_dir[0] == '\0' && cuname[0] != '/')
3001 : : {
3002 : : // This is a common symptom for dwz-compressed debug files,
3003 : : // where the altdebug file cannot be resolved.
3004 [ # # ]: 0 : if (verbose > 3)
3005 : 0 : obatched(clog) << "skipping cu=" << cuname << " due to empty comp_dir" << endl;
3006 : 0 : continue;
3007 : : }
3008 : :
3009 [ + + ]: 32480 : for (size_t f = 1; f < nfiles; f++)
3010 : : {
3011 : 30400 : const char *hat = dwarf_filesrc (files, f, NULL, NULL);
3012 [ - + ]: 30400 : if (hat == NULL)
3013 : 0 : continue;
3014 : :
3015 [ + + - + ]: 58028 : if (string(hat) == "<built-in>") // gcc intrinsics, don't bother record
3016 : 0 : continue;
3017 : :
3018 [ + + ]: 30400 : string waldo;
3019 [ + + ]: 30400 : if (hat[0] == '/') // absolute
3020 [ - + ]: 19286 : waldo = (string (hat));
3021 [ + - ]: 11114 : else if (comp_dir[0] != '\0') // comp_dir relative
3022 [ - + - + : 19456 : waldo = (string (comp_dir) + string("/") + string (hat));
- + - + +
+ ]
3023 : : else
3024 : : {
3025 [ # # ]: 0 : if (verbose > 3)
3026 : 0 : obatched(clog) << "skipping hat=" << hat << " due to empty comp_dir" << endl;
3027 [ # # ]: 0 : continue;
3028 : : }
3029 : :
3030 : : // NB: this is the 'waldo' that a dbginfo client will have
3031 : : // to supply for us to give them the file The comp_dir
3032 : : // prefixing is a definite complication. Otherwise we'd
3033 : : // have to return a setof comp_dirs (one per CU!) with
3034 : : // corresponding filesrc[] names, instead of one absolute
3035 : : // resoved set. Maybe we'll have to do that anyway. XXX
3036 : :
3037 [ + + ]: 30400 : if (verbose > 4)
3038 [ - + ]: 100 : obatched(clog) << waldo
3039 [ - + ]: 50 : << (debug_sourcefiles.find(waldo)==debug_sourcefiles.end() ? " new" : " dup") << endl;
3040 : :
3041 [ + - ]: 30400 : debug_sourcefiles.insert (waldo);
3042 : 30400 : }
3043 : : }
3044 : :
3045 : 210 : dwarf_end(dbg);
3046 [ + + ]: 210 : if (altdbg)
3047 : 40 : dwarf_end(altdbg);
3048 [ + + ]: 210 : if (altdbg_fd >= 0)
3049 : 40 : close(altdbg_fd);
3050 : : }
3051 : :
3052 : :
3053 : :
3054 : : static void
3055 : 1034 : elf_classify (int fd, bool &executable_p, bool &debuginfo_p, string &buildid, set<string>& debug_sourcefiles)
3056 : : {
3057 : 1034 : Elf *elf = elf_begin (fd, ELF_C_READ_MMAP_PRIVATE, NULL);
3058 [ + - ]: 1034 : if (elf == NULL)
3059 : : return;
3060 : :
3061 : 1034 : try // catch our types of errors and clean up the Elf* object
3062 : : {
3063 [ + - + + ]: 1034 : if (elf_kind (elf) != ELF_K_ELF)
3064 : : {
3065 [ + - ]: 620 : elf_end (elf);
3066 : 620 : return;
3067 : : }
3068 : :
3069 : 414 : GElf_Ehdr ehdr_storage;
3070 [ + - ]: 414 : GElf_Ehdr *ehdr = gelf_getehdr (elf, &ehdr_storage);
3071 [ - + ]: 414 : if (ehdr == NULL)
3072 : : {
3073 [ # # ]: 0 : elf_end (elf);
3074 : : return;
3075 : : }
3076 : 414 : auto elf_type = ehdr->e_type;
3077 : :
3078 : 414 : const void *build_id; // elfutils-owned memory
3079 [ + - ]: 414 : ssize_t sz = dwelf_elf_gnu_build_id (elf, & build_id);
3080 [ - + ]: 413 : if (sz <= 0)
3081 : : {
3082 : : // It's not a diagnostic-worthy error for an elf file to lack build-id.
3083 : : // It might just be very old.
3084 [ # # ]: 0 : elf_end (elf);
3085 : : return;
3086 : : }
3087 : :
3088 : : // build_id is a raw byte array; convert to hexadecimal *lowercase*
3089 : 413 : unsigned char* build_id_bytes = (unsigned char*) build_id;
3090 [ + + ]: 8678 : for (ssize_t idx=0; idx<sz; idx++)
3091 : : {
3092 [ + - ]: 8264 : buildid += "0123456789abcdef"[build_id_bytes[idx] >> 4];
3093 [ + - ]: 16529 : buildid += "0123456789abcdef"[build_id_bytes[idx] & 0xf];
3094 : : }
3095 : :
3096 : : // now decide whether it's an executable - namely, any allocatable section has
3097 : : // PROGBITS;
3098 [ + + ]: 414 : if (elf_type == ET_EXEC || elf_type == ET_DYN)
3099 : : {
3100 : 364 : size_t shnum;
3101 [ + - ]: 364 : int rc = elf_getshdrnum (elf, &shnum);
3102 [ - + ]: 363 : if (rc < 0)
3103 [ # # # # ]: 0 : throw elfutils_exception(rc, "getshdrnum");
3104 : :
3105 : 363 : executable_p = false;
3106 [ + + ]: 6984 : for (size_t sc = 0; sc < shnum; sc++)
3107 : : {
3108 [ + - ]: 6808 : Elf_Scn *scn = elf_getscn (elf, sc);
3109 [ - + ]: 6808 : if (scn == NULL)
3110 : 0 : continue;
3111 : :
3112 : 6808 : GElf_Shdr shdr_mem;
3113 [ + - ]: 6808 : GElf_Shdr *shdr = gelf_getshdr (scn, &shdr_mem);
3114 [ - + ]: 6809 : if (shdr == NULL)
3115 : 0 : continue;
3116 : :
3117 : : // allocated (loadable / vm-addr-assigned) section with available content?
3118 [ + + + + ]: 6809 : if ((shdr->sh_type == SHT_PROGBITS) && (shdr->sh_flags & SHF_ALLOC))
3119 : : {
3120 [ + + ]: 188 : if (verbose > 4)
3121 [ + - + - : 32 : obatched(clog) << "executable due to SHF_ALLOC SHT_PROGBITS sc=" << sc << endl;
+ - ]
3122 : 188 : executable_p = true;
3123 : 188 : break; // no need to keep looking for others
3124 : : }
3125 : : } // iterate over sections
3126 : : } // executable_p classification
3127 : :
3128 : : // now decide whether it's a debuginfo - namely, if it has any .debug* or .zdebug* sections
3129 : : // logic mostly stolen from fweimer@redhat.com's elfclassify drafts
3130 : 414 : size_t shstrndx;
3131 [ + - ]: 414 : int rc = elf_getshdrstrndx (elf, &shstrndx);
3132 [ - + ]: 414 : if (rc < 0)
3133 [ # # # # ]: 0 : throw elfutils_exception(rc, "getshdrstrndx");
3134 : :
3135 : : Elf_Scn *scn = NULL;
3136 : : bool symtab_p = false;
3137 : : bool bits_alloc_p = false;
3138 : 21330 : while (true)
3139 : : {
3140 [ + - ]: 10872 : scn = elf_nextscn (elf, scn);
3141 [ + + ]: 10856 : if (scn == NULL)
3142 : : break;
3143 : 10652 : GElf_Shdr shdr_storage;
3144 [ + - ]: 10652 : GElf_Shdr *shdr = gelf_getshdr (scn, &shdr_storage);
3145 [ - + ]: 10652 : if (shdr == NULL)
3146 : : break;
3147 [ + - ]: 10652 : const char *section_name = elf_strptr (elf, shstrndx, shdr->sh_name);
3148 [ - + ]: 10668 : if (section_name == NULL)
3149 : : break;
3150 [ + + ]: 10668 : if (startswith (section_name, ".debug_line") ||
3151 [ - + ]: 10458 : startswith (section_name, ".zdebug_line"))
3152 : : {
3153 : 210 : debuginfo_p = true;
3154 [ + - ]: 210 : if (scan_source_info)
3155 : 210 : dwarf_extract_source_paths (elf, debug_sourcefiles);
3156 : : break; // expecting only one .*debug_line, so no need to look for others
3157 : : }
3158 [ + + ]: 10458 : else if (startswith (section_name, ".debug_") ||
3159 [ + # ]: 9775 : startswith (section_name, ".zdebug_"))
3160 : : {
3161 : 668 : debuginfo_p = true;
3162 : : // NB: don't break; need to parse .debug_line for sources
3163 : : }
3164 [ + + ]: 9790 : else if (shdr->sh_type == SHT_SYMTAB)
3165 : : {
3166 : : symtab_p = true;
3167 : : }
3168 : 9766 : else if (shdr->sh_type != SHT_NOBITS
3169 [ + + ]: 9766 : && shdr->sh_type != SHT_NOTE
3170 [ + + ]: 4698 : && (shdr->sh_flags & SHF_ALLOC) != 0)
3171 : : {
3172 : 10458 : bits_alloc_p = true;
3173 : : }
3174 : 10458 : }
3175 : :
3176 : : // For more expansive elf/split-debuginfo classification, we
3177 : : // want to identify as debuginfo "strip -s"-produced files
3178 : : // without .debug_info* (like libicudata), but we don't want to
3179 : : // identify "strip -g" executables (with .symtab left there).
3180 [ - + ]: 414 : if (symtab_p && !bits_alloc_p)
3181 : 0 : debuginfo_p = true;
3182 : : }
3183 [ # # ]: 0 : catch (const reportable_exception& e)
3184 : : {
3185 [ # # ]: 0 : e.report(clog);
3186 : 0 : }
3187 : 414 : elf_end (elf);
3188 : : }
3189 : :
3190 : :
3191 : : // Intern the given file name in two parts (dirname & basename) and
3192 : : // return the resulting file's id.
3193 : : static int64_t
3194 : 8632 : register_file_name(sqlite_ps& ps_upsert_fileparts,
3195 : : sqlite_ps& ps_upsert_file,
3196 : : sqlite_ps& ps_lookup_file,
3197 : : const string& name)
3198 : : {
3199 : 8632 : std::size_t slash = name.rfind('/');
3200 [ + + ]: 8632 : string dirname, basename;
3201 [ + + ]: 8632 : if (slash == std::string::npos)
3202 : : {
3203 [ + - ]: 90 : dirname = "";
3204 [ + - ]: 90 : basename = name;
3205 : : }
3206 : : else
3207 : : {
3208 [ + - - + ]: 8542 : dirname = name.substr(0, slash);
3209 [ + - - + : 8541 : basename = name.substr(slash+1);
- - ]
3210 : : }
3211 : :
3212 : : // intern the two substrings
3213 : 8631 : ps_upsert_fileparts
3214 [ + - ]: 8631 : .reset()
3215 [ + - ]: 8632 : .bind(1, dirname)
3216 [ + - ]: 8632 : .step_ok_done();
3217 : 8632 : ps_upsert_fileparts
3218 [ + - ]: 8632 : .reset()
3219 [ + - ]: 8632 : .bind(1, basename)
3220 [ + - ]: 8632 : .step_ok_done();
3221 : :
3222 : : // intern the tuple
3223 : 8632 : ps_upsert_file
3224 [ + - ]: 8632 : .reset()
3225 [ + - ]: 8632 : .bind(1, dirname)
3226 [ + - ]: 8631 : .bind(2, basename)
3227 [ + - ]: 8632 : .step_ok_done();
3228 : :
3229 : : // look up the tuple's id
3230 : 8632 : ps_lookup_file
3231 [ + - ]: 8632 : .reset()
3232 [ + - ]: 8632 : .bind(1, dirname)
3233 [ + - ]: 8631 : .bind(2, basename);
3234 [ + - ]: 8632 : int rc = ps_lookup_file.step();
3235 [ - + - - : 8632 : if (rc != SQLITE_ROW) throw sqlite_exception(rc, "step");
- - ]
3236 : :
3237 [ + - ]: 8632 : int64_t id = sqlite3_column_int64 (ps_lookup_file, 0);
3238 [ + - ]: 8632 : ps_lookup_file.reset();
3239 [ + + ]: 8632 : return id;
3240 [ + + ]: 16536 : }
3241 : :
3242 : :
3243 : :
3244 : : static void
3245 : 728 : scan_source_file (const string& rps, const stat_t& st,
3246 : : sqlite_ps& ps_upsert_buildids,
3247 : : sqlite_ps& ps_upsert_fileparts,
3248 : : sqlite_ps& ps_upsert_file,
3249 : : sqlite_ps& ps_lookup_file,
3250 : : sqlite_ps& ps_upsert_de,
3251 : : sqlite_ps& ps_upsert_s,
3252 : : sqlite_ps& ps_query,
3253 : : sqlite_ps& ps_scan_done,
3254 : : unsigned& fts_cached,
3255 : : unsigned& fts_executable,
3256 : : unsigned& fts_debuginfo,
3257 : : unsigned& fts_sourcefiles)
3258 : : {
3259 : 728 : int64_t fileid = register_file_name(ps_upsert_fileparts, ps_upsert_file, ps_lookup_file, rps);
3260 : :
3261 : : /* See if we know of it already. */
3262 : 728 : int rc = ps_query
3263 : 728 : .reset()
3264 : 728 : .bind(1, fileid)
3265 : 728 : .bind(2, st.st_mtime)
3266 : 728 : .step();
3267 : 728 : ps_query.reset();
3268 [ + + ]: 728 : if (rc == SQLITE_ROW) // i.e., a result, as opposed to DONE (no results)
3269 : : // no need to recheck a file/version we already know
3270 : : // specifically, no need to elf-begin a file we already determined is non-elf
3271 : : // (so is stored with buildid=NULL)
3272 : : {
3273 : 368 : fts_cached++;
3274 : 368 : return;
3275 : : }
3276 : :
3277 : 360 : bool executable_p = false, debuginfo_p = false; // E and/or D
3278 [ + - ]: 360 : string buildid;
3279 [ + - ]: 360 : set<string> sourcefiles;
3280 : :
3281 [ + - ]: 360 : int fd = open (rps.c_str(), O_RDONLY);
3282 : 360 : try
3283 : : {
3284 [ + - ]: 360 : if (fd >= 0)
3285 [ + - ]: 360 : elf_classify (fd, executable_p, debuginfo_p, buildid, sourcefiles);
3286 : : else
3287 [ # # # # : 0 : throw libc_exception(errno, string("open ") + rps);
# # # # ]
3288 [ + - + - : 720 : add_metric ("scanned_bytes_total","source","file",
+ - - + -
+ - - -
- ]
3289 [ + - ]: 360 : st.st_size);
3290 [ + - + - : 720 : inc_metric ("scanned_files_total","source","file");
+ - + - -
+ - + - -
- - ]
3291 : : }
3292 : : // NB: we catch exceptions here too, so that we can
3293 : : // cache the corrupt-elf case (!executable_p &&
3294 : : // !debuginfo_p) just below, just as if we had an
3295 : : // EPERM error from open(2).
3296 [ - - ]: 0 : catch (const reportable_exception& e)
3297 : : {
3298 [ - - ]: 0 : e.report(clog);
3299 : 0 : }
3300 : :
3301 [ + - ]: 360 : if (fd >= 0)
3302 [ + - ]: 360 : close (fd);
3303 : :
3304 [ + + ]: 360 : if (buildid == "")
3305 : : {
3306 : : // no point storing an elf file without buildid
3307 : 298 : executable_p = false;
3308 : 298 : debuginfo_p = false;
3309 : : }
3310 : : else
3311 : : {
3312 : : // register this build-id in the interning table
3313 : 62 : ps_upsert_buildids
3314 [ + - ]: 62 : .reset()
3315 [ + - ]: 62 : .bind(1, buildid)
3316 [ + - ]: 62 : .step_ok_done();
3317 : : }
3318 : :
3319 [ + + ]: 360 : if (executable_p)
3320 : 38 : fts_executable ++;
3321 [ + + ]: 360 : if (debuginfo_p)
3322 : 38 : fts_debuginfo ++;
3323 [ + + + + ]: 360 : if (executable_p || debuginfo_p)
3324 : : {
3325 : 62 : ps_upsert_de
3326 [ + - ]: 62 : .reset()
3327 [ + - ]: 62 : .bind(1, buildid)
3328 [ + + + - ]: 86 : .bind(2, debuginfo_p ? 1 : 0)
3329 [ + + + - ]: 86 : .bind(3, executable_p ? 1 : 0)
3330 [ + - ]: 62 : .bind(4, fileid)
3331 [ + - ]: 62 : .bind(5, st.st_mtime)
3332 [ + - ]: 62 : .step_ok_done();
3333 : : }
3334 [ + + ]: 360 : if (executable_p)
3335 [ + - + - : 76 : inc_metric("found_executable_total","source","files");
+ - + - -
+ - + - -
- - ]
3336 [ + + ]: 360 : if (debuginfo_p)
3337 [ + - + - : 76 : inc_metric("found_debuginfo_total","source","files");
+ - + - -
+ - + - -
- - ]
3338 : :
3339 [ + + + - ]: 398 : if (sourcefiles.size() && buildid != "")
3340 : : {
3341 : 38 : fts_sourcefiles += sourcefiles.size();
3342 : :
3343 [ + + ]: 3106 : for (auto&& dwarfsrc : sourcefiles)
3344 : : {
3345 [ + - ]: 3068 : char *srp = realpath(dwarfsrc.c_str(), NULL);
3346 [ + + ]: 3068 : if (srp == NULL) // also if DWZ unresolved dwarfsrc=""
3347 : 36 : continue; // unresolvable files are not a serious problem
3348 : : // throw libc_exception(errno, "fts/file realpath " + srcpath);
3349 [ + - ]: 3032 : string srps = string(srp);
3350 : 3032 : free (srp);
3351 : :
3352 : 3032 : struct stat sfs;
3353 : 3032 : rc = stat(srps.c_str(), &sfs);
3354 [ - + ]: 3032 : if (rc != 0)
3355 [ # # ]: 0 : continue;
3356 : :
3357 [ + - ]: 3032 : if (verbose > 2)
3358 [ + - + - : 9096 : obatched(clog) << "recorded buildid=" << buildid << " file=" << srps
- - ]
3359 [ + - + - : 3032 : << " mtime=" << sfs.st_mtime
+ - + - ]
3360 [ + - + - : 3032 : << " as source " << dwarfsrc << endl;
+ - ]
3361 : :
3362 : : // PR25548: store canonicalized dwarfsrc path
3363 [ + - ]: 3032 : string dwarfsrc_canon = canon_pathname (dwarfsrc);
3364 [ + + ]: 3032 : if (dwarfsrc_canon != dwarfsrc)
3365 : : {
3366 [ + + ]: 508 : if (verbose > 3)
3367 [ + - + - : 20 : obatched(clog) << "canonicalized src=" << dwarfsrc << " alias=" << dwarfsrc_canon << endl;
+ - + - +
- ]
3368 : : }
3369 : :
3370 [ + - ]: 3032 : int64_t fileid1 = register_file_name (ps_upsert_fileparts, ps_upsert_file, ps_lookup_file, dwarfsrc_canon);
3371 [ + - ]: 3032 : int64_t fileid2 = register_file_name (ps_upsert_fileparts, ps_upsert_file, ps_lookup_file, srps);
3372 : :
3373 : 3032 : ps_upsert_s
3374 [ + - ]: 3032 : .reset()
3375 [ + - ]: 3032 : .bind(1, buildid)
3376 [ + - ]: 3032 : .bind(2, fileid1)
3377 [ + - ]: 3032 : .bind(3, fileid2)
3378 [ + - ]: 3032 : .bind(4, sfs.st_mtime)
3379 [ + - ]: 3032 : .step_ok_done();
3380 : :
3381 [ + - + - : 6064 : inc_metric("found_sourcerefs_total","source","files");
+ - + - -
+ - + + -
- - - - -
- ]
3382 [ + - ]: 6100 : }
3383 : : }
3384 : :
3385 : 360 : ps_scan_done
3386 [ + - ]: 360 : .reset()
3387 [ + - ]: 360 : .bind(1, fileid)
3388 [ + - ]: 360 : .bind(2, st.st_mtime)
3389 [ + - ]: 360 : .bind(3, st.st_size)
3390 [ + - ]: 360 : .step_ok_done();
3391 : :
3392 [ + - ]: 360 : if (verbose > 2)
3393 [ + - + - ]: 1080 : obatched(clog) << "recorded buildid=" << buildid << " file=" << rps
3394 [ + - + - : 360 : << " mtime=" << st.st_mtime << " atype="
+ - + - ]
3395 : : << (executable_p ? "E" : "")
3396 [ + - + + : 1004 : << (debuginfo_p ? "D" : "") << endl;
+ - + + +
- + - ]
3397 [ + + ]: 422 : }
3398 : :
3399 : :
3400 : :
3401 : :
3402 : :
3403 : : // Analyze given archive file of given age; record buildids / exec/debuginfo-ness of its
3404 : : // constituent files with given upsert statements.
3405 : : static void
3406 : 362 : archive_classify (const string& rps, string& archive_extension, int64_t archiveid,
3407 : : sqlite_ps& ps_upsert_buildids, sqlite_ps& ps_upsert_fileparts, sqlite_ps& ps_upsert_file,
3408 : : sqlite_ps& ps_lookup_file,
3409 : : sqlite_ps& ps_upsert_de, sqlite_ps& ps_upsert_sref, sqlite_ps& ps_upsert_sdef,
3410 : : time_t mtime,
3411 : : unsigned& fts_executable, unsigned& fts_debuginfo, unsigned& fts_sref, unsigned& fts_sdef,
3412 : : bool& fts_sref_complete_p)
3413 : : {
3414 : 362 : string archive_decoder = "/dev/null";
3415 [ + + ]: 893 : for (auto&& arch : scan_archives)
3416 [ + + ]: 532 : if (string_endswith(rps, arch.first))
3417 : : {
3418 [ + - ]: 362 : archive_extension = arch.first;
3419 [ + - ]: 893 : archive_decoder = arch.second;
3420 : : }
3421 : :
3422 : 361 : FILE* fp;
3423 : 361 : defer_dtor<FILE*,int>::dtor_fn dfn;
3424 [ + + ]: 361 : if (archive_decoder != "cat")
3425 : : {
3426 [ + - + - : 64 : string popen_cmd = archive_decoder + " " + shell_escape(rps);
+ - - + -
- - - ]
3427 [ + - ]: 32 : fp = popen (popen_cmd.c_str(), "r"); // "e" O_CLOEXEC?
3428 : 32 : dfn = pclose;
3429 [ - + ]: 32 : if (fp == NULL)
3430 [ # # # # : 0 : throw libc_exception (errno, string("popen ") + popen_cmd);
# # # # ]
3431 : 32 : }
3432 : : else
3433 : : {
3434 [ + - ]: 329 : fp = fopen (rps.c_str(), "r");
3435 : 330 : dfn = fclose;
3436 [ - + ]: 330 : if (fp == NULL)
3437 [ # # # # : 0 : throw libc_exception (errno, string("fopen ") + rps);
# # # # ]
3438 : : }
3439 : 362 : defer_dtor<FILE*,int> fp_closer (fp, dfn);
3440 : :
3441 : 362 : struct archive *a;
3442 [ + - ]: 362 : a = archive_read_new();
3443 [ - + ]: 362 : if (a == NULL)
3444 [ # # # # ]: 0 : throw archive_exception("cannot create archive reader");
3445 : 362 : defer_dtor<struct archive*,int> archive_closer (a, archive_read_free);
3446 : :
3447 [ + - ]: 362 : int rc = archive_read_support_format_all(a);
3448 [ - + ]: 362 : if (rc != ARCHIVE_OK)
3449 [ # # # # ]: 0 : throw archive_exception(a, "cannot select all formats");
3450 [ + - ]: 362 : rc = archive_read_support_filter_all(a);
3451 [ - + ]: 362 : if (rc != ARCHIVE_OK)
3452 [ # # # # ]: 0 : throw archive_exception(a, "cannot select all filters");
3453 : :
3454 [ + - ]: 362 : rc = archive_read_open_FILE (a, fp);
3455 [ - + ]: 362 : if (rc != ARCHIVE_OK)
3456 : : {
3457 [ # # # # : 0 : obatched(clog) << "cannot open archive from pipe " << rps << endl;
# # ]
3458 [ # # # # ]: 0 : throw archive_exception(a, "cannot open archive from pipe");
3459 : : }
3460 : :
3461 [ + + ]: 362 : if (verbose > 3)
3462 [ + - + - : 692 : obatched(clog) << "libarchive scanning " << rps << " id " << archiveid << endl;
+ - + - +
- ]
3463 : :
3464 : : bool any_exceptions = false;
3465 : 2564 : while(1) // parse archive entries
3466 : : {
3467 [ + - ]: 2564 : if (interrupted)
3468 : : break;
3469 : :
3470 : 2564 : try
3471 : : {
3472 : 2564 : struct archive_entry *e;
3473 [ + - ]: 2564 : rc = archive_read_next_header (a, &e);
3474 [ + + ]: 2564 : if (rc != ARCHIVE_OK)
3475 : : break;
3476 : :
3477 [ + - + + ]: 2202 : if (! S_ISREG(archive_entry_mode (e))) // skip non-files completely
3478 : 1528 : continue;
3479 : :
3480 [ + - ]: 674 : string fn = canonicalized_archive_entry_pathname (e);
3481 : :
3482 [ + + ]: 674 : if (verbose > 3)
3483 [ + - + - : 1268 : obatched(clog) << "libarchive checking " << fn << endl;
+ - - - ]
3484 : :
3485 : : // extract this file to a temporary file
3486 : 674 : char* tmppath = NULL;
3487 : 674 : rc = asprintf (&tmppath, "%s/debuginfod.XXXXXX", tmpdir.c_str());
3488 [ - + ]: 674 : if (rc < 0)
3489 [ # # # # ]: 0 : throw libc_exception (ENOMEM, "cannot allocate tmppath");
3490 : 674 : defer_dtor<void*,void> tmmpath_freer (tmppath, free);
3491 [ + - ]: 674 : int fd = mkstemp (tmppath);
3492 [ - + ]: 674 : if (fd < 0)
3493 [ # # # # ]: 0 : throw libc_exception (errno, "cannot create temporary file");
3494 : 674 : unlink (tmppath); // unlink now so OS will release the file as soon as we close the fd
3495 : 674 : defer_dtor<int,int> minifd_closer (fd, close);
3496 : :
3497 [ + - ]: 674 : rc = archive_read_data_into_fd (a, fd);
3498 [ - + ]: 674 : if (rc != ARCHIVE_OK)
3499 [ # # # # ]: 0 : throw archive_exception(a, "cannot extract file");
3500 : :
3501 : : // finally ... time to run elf_classify on this bad boy and update the database
3502 : 674 : bool executable_p = false, debuginfo_p = false;
3503 [ + - ]: 674 : string buildid;
3504 [ + - ]: 674 : set<string> sourcefiles;
3505 [ + - ]: 674 : elf_classify (fd, executable_p, debuginfo_p, buildid, sourcefiles);
3506 : : // NB: might throw
3507 : :
3508 [ + + ]: 674 : if (buildid != "") // intern buildid
3509 : : {
3510 : 352 : ps_upsert_buildids
3511 [ + - ]: 352 : .reset()
3512 [ + - ]: 352 : .bind(1, buildid)
3513 [ + - ]: 352 : .step_ok_done();
3514 : : }
3515 : :
3516 [ + - ]: 674 : int64_t fileid = register_file_name (ps_upsert_fileparts, ps_upsert_file, ps_lookup_file, fn);
3517 : :
3518 [ + + ]: 674 : if (sourcefiles.size() > 0) // sref records needed
3519 : : {
3520 : : // NB: we intern each source file once. Once raw, as it
3521 : : // appears in the DWARF file list coming back from
3522 : : // elf_classify() - because it'll end up in the
3523 : : // _norm.artifactsrc column. We don't also put another
3524 : : // version with a '.' at the front, even though that's
3525 : : // how rpm/cpio packs names, because we hide that from
3526 : : // the database for storage efficiency.
3527 : :
3528 [ + + ]: 616 : for (auto&& s : sourcefiles)
3529 : : {
3530 [ - + ]: 464 : if (s == "")
3531 : : {
3532 : 0 : fts_sref_complete_p = false;
3533 : 0 : continue;
3534 : : }
3535 : :
3536 : : // PR25548: store canonicalized source path
3537 : 464 : const string& dwarfsrc = s;
3538 [ + - ]: 464 : string dwarfsrc_canon = canon_pathname (dwarfsrc);
3539 [ + + ]: 464 : if (dwarfsrc_canon != dwarfsrc)
3540 : : {
3541 [ + - ]: 32 : if (verbose > 3)
3542 [ + - + - : 64 : obatched(clog) << "canonicalized src=" << dwarfsrc << " alias=" << dwarfsrc_canon << endl;
+ - + - +
- - - ]
3543 : : }
3544 : :
3545 [ + - ]: 464 : int64_t srcfileid = register_file_name(ps_upsert_fileparts, ps_upsert_file, ps_lookup_file,
3546 : : dwarfsrc_canon);
3547 : :
3548 : 464 : ps_upsert_sref
3549 [ + - ]: 464 : .reset()
3550 [ + - ]: 464 : .bind(1, buildid)
3551 [ + - ]: 464 : .bind(2, srcfileid)
3552 [ + - ]: 464 : .step_ok_done();
3553 : :
3554 [ + - ]: 464 : fts_sref ++;
3555 : 464 : }
3556 : : }
3557 : :
3558 [ + + ]: 674 : if (executable_p)
3559 : 150 : fts_executable ++;
3560 [ + + ]: 674 : if (debuginfo_p)
3561 : 202 : fts_debuginfo ++;
3562 : :
3563 [ + + + + ]: 674 : if (executable_p || debuginfo_p)
3564 : : {
3565 : 352 : ps_upsert_de
3566 [ + - ]: 352 : .reset()
3567 [ + - ]: 352 : .bind(1, buildid)
3568 [ + + + - ]: 502 : .bind(2, debuginfo_p ? 1 : 0)
3569 [ + + + - ]: 554 : .bind(3, executable_p ? 1 : 0)
3570 [ + - ]: 352 : .bind(4, archiveid)
3571 [ + - ]: 352 : .bind(5, mtime)
3572 [ + - ]: 352 : .bind(6, fileid)
3573 [ + - ]: 352 : .step_ok_done();
3574 : : }
3575 : : else // potential source - sdef record
3576 : : {
3577 : 322 : fts_sdef ++;
3578 : 322 : ps_upsert_sdef
3579 [ + - ]: 322 : .reset()
3580 [ + - ]: 322 : .bind(1, archiveid)
3581 [ + - ]: 322 : .bind(2, mtime)
3582 [ + - ]: 322 : .bind(3, fileid)
3583 [ + - ]: 322 : .step_ok_done();
3584 : : }
3585 : :
3586 [ + - + + : 674 : if ((verbose > 2) && (executable_p || debuginfo_p))
+ + ]
3587 [ + - + - ]: 1056 : obatched(clog) << "recorded buildid=" << buildid << " rpm=" << rps << " file=" << fn
3588 [ + - + - : 352 : << " mtime=" << mtime << " atype="
+ - + - +
- + - ]
3589 : : << (executable_p ? "E" : "")
3590 : : << (debuginfo_p ? "D" : "")
3591 [ + - + + : 704 : << " sourcefiles=" << sourcefiles.size() << endl;
+ - + + +
- + - + -
+ - ]
3592 : :
3593 [ + + + + ]: 1532 : }
3594 [ - - ]: 0 : catch (const reportable_exception& e)
3595 : : {
3596 [ - - ]: 0 : e.report(clog);
3597 : 0 : any_exceptions = true;
3598 : : // NB: but we allow the libarchive iteration to continue, in
3599 : : // case we can still gather some useful information. That
3600 : : // would allow some webapi queries to work, until later when
3601 : : // this archive is rescanned. (Its vitals won't go into the
3602 : : // _file_mtime_scanned table until after a successful scan.)
3603 : 0 : }
3604 : : }
3605 : :
3606 [ - + ]: 362 : if (any_exceptions)
3607 [ # # # # ]: 0 : throw reportable_exception("exceptions encountered during archive scan");
3608 [ + + ]: 378 : }
3609 : :
3610 : :
3611 : :
3612 : : // scan for archive files such as .rpm
3613 : : static void
3614 : 702 : scan_archive_file (const string& rps, const stat_t& st,
3615 : : sqlite_ps& ps_upsert_buildids,
3616 : : sqlite_ps& ps_upsert_fileparts,
3617 : : sqlite_ps& ps_upsert_file,
3618 : : sqlite_ps& ps_lookup_file,
3619 : : sqlite_ps& ps_upsert_de,
3620 : : sqlite_ps& ps_upsert_sref,
3621 : : sqlite_ps& ps_upsert_sdef,
3622 : : sqlite_ps& ps_query,
3623 : : sqlite_ps& ps_scan_done,
3624 : : unsigned& fts_cached,
3625 : : unsigned& fts_executable,
3626 : : unsigned& fts_debuginfo,
3627 : : unsigned& fts_sref,
3628 : : unsigned& fts_sdef)
3629 : : {
3630 : : // intern the archive file name
3631 : 702 : int64_t archiveid = register_file_name (ps_upsert_fileparts, ps_upsert_file, ps_lookup_file, rps);
3632 : :
3633 : : /* See if we know of it already. */
3634 : 702 : int rc = ps_query
3635 : 702 : .reset()
3636 : 702 : .bind(1, archiveid)
3637 : 702 : .bind(2, st.st_mtime)
3638 : 702 : .step();
3639 : 702 : ps_query.reset();
3640 [ + + ]: 702 : if (rc == SQLITE_ROW) // i.e., a result, as opposed to DONE (no results)
3641 : : // no need to recheck a file/version we already know
3642 : : // specifically, no need to parse this archive again, since we already have
3643 : : // it as a D or E or S record,
3644 : : // (so is stored with buildid=NULL)
3645 : : {
3646 : 340 : fts_cached ++;
3647 : 340 : return;
3648 : : }
3649 : :
3650 : : // extract the archive contents
3651 : 362 : unsigned my_fts_executable = 0, my_fts_debuginfo = 0, my_fts_sref = 0, my_fts_sdef = 0;
3652 : 362 : bool my_fts_sref_complete_p = true;
3653 : 362 : bool any_exceptions = false;
3654 : 362 : try
3655 : : {
3656 [ + - ]: 362 : string archive_extension;
3657 : 362 : archive_classify (rps, archive_extension, archiveid,
3658 : : ps_upsert_buildids, ps_upsert_fileparts, ps_upsert_file, ps_lookup_file,
3659 : : ps_upsert_de, ps_upsert_sref, ps_upsert_sdef, // dalt
3660 [ + - ]: 362 : st.st_mtime,
3661 : : my_fts_executable, my_fts_debuginfo, my_fts_sref, my_fts_sdef,
3662 : : my_fts_sref_complete_p);
3663 [ + - + - : 724 : add_metric ("scanned_bytes_total","source",archive_extension + " archive",
+ - - + +
+ - - -
- ]
3664 [ + - ]: 362 : st.st_size);
3665 [ + - + - : 724 : inc_metric ("scanned_files_total","source",archive_extension + " archive");
+ - + - -
+ + + - -
- - ]
3666 [ + - + - : 724 : add_metric("found_debuginfo_total","source",archive_extension + " archive",
+ - + - -
+ + + - -
- - ]
3667 : : my_fts_debuginfo);
3668 [ + - + - : 724 : add_metric("found_executable_total","source",archive_extension + " archive",
+ - + - -
+ + + - -
- - ]
3669 : : my_fts_executable);
3670 [ + - + - : 740 : add_metric("found_sourcerefs_total","source",archive_extension + " archive",
+ - + - -
+ + + - +
- - - - -
- ]
3671 : : my_fts_sref);
3672 : 362 : }
3673 [ - - ]: 0 : catch (const reportable_exception& e)
3674 : : {
3675 [ - - ]: 0 : e.report(clog);
3676 : 0 : any_exceptions = true;
3677 : 0 : }
3678 : :
3679 [ + - ]: 362 : if (verbose > 2)
3680 [ + - ]: 1086 : obatched(clog) << "scanned archive=" << rps
3681 [ + - + - ]: 362 : << " mtime=" << st.st_mtime
3682 [ + - ]: 362 : << " executables=" << my_fts_executable
3683 [ + - + - ]: 362 : << " debuginfos=" << my_fts_debuginfo
3684 [ + - + - ]: 362 : << " srefs=" << my_fts_sref
3685 [ + - + - ]: 362 : << " sdefs=" << my_fts_sdef
3686 [ + - + - : 362 : << " exceptions=" << any_exceptions
+ - + - ]
3687 : 362 : << endl;
3688 : :
3689 : 362 : fts_executable += my_fts_executable;
3690 : 362 : fts_debuginfo += my_fts_debuginfo;
3691 : 362 : fts_sref += my_fts_sref;
3692 : 362 : fts_sdef += my_fts_sdef;
3693 : :
3694 [ - + ]: 362 : if (any_exceptions)
3695 [ # # # # ]: 0 : throw reportable_exception("exceptions encountered during archive scan");
3696 : :
3697 [ + - ]: 362 : if (my_fts_sref_complete_p) // leave incomplete?
3698 : 362 : ps_scan_done
3699 : 362 : .reset()
3700 : 362 : .bind(1, archiveid)
3701 : 362 : .bind(2, st.st_mtime)
3702 : 362 : .bind(3, st.st_size)
3703 : 362 : .step_ok_done();
3704 : : }
3705 : :
3706 : :
3707 : :
3708 : : ////////////////////////////////////////////////////////////////////////
3709 : :
3710 : :
3711 : :
3712 : : // The thread that consumes file names off of the scanq. We hold
3713 : : // the persistent sqlite_ps's at this level and delegate file/archive
3714 : : // scanning to other functions.
3715 : : static void
3716 : 256 : scan ()
3717 : : {
3718 : : // all the prepared statements fit to use, the _f_ set:
3719 [ + - + - : 512 : sqlite_ps ps_f_upsert_buildids (db, "file-buildids-intern", "insert or ignore into " BUILDIDS "_buildids VALUES (NULL, ?);");
+ - - - ]
3720 [ + - + - : 512 : sqlite_ps ps_f_upsert_fileparts (db, "file-fileparts-intern", "insert or ignore into " BUILDIDS "_fileparts VALUES (NULL, ?);");
+ - + - -
- ]
3721 [ + - ]: 256 : sqlite_ps ps_f_upsert_file (db, "file-file-intern", "insert or ignore into " BUILDIDS "_files VALUES (NULL, \n"
3722 : : "(select id from " BUILDIDS "_fileparts where name = ?),\n"
3723 [ + - + - : 512 : "(select id from " BUILDIDS "_fileparts where name = ?));");
+ - + - -
- ]
3724 [ + - ]: 256 : sqlite_ps ps_f_lookup_file (db, "file-file-lookup",
3725 : : "select f.id\n"
3726 : : " from " BUILDIDS "_files f, " BUILDIDS "_fileparts p1, " BUILDIDS "_fileparts p2 \n"
3727 [ + - + - : 512 : " where f.dirname = p1.id and f.basename = p2.id and p1.name = ? and p2.name = ?;\n");
+ - + - -
- ]
3728 [ + - ]: 256 : sqlite_ps ps_f_upsert_de (db, "file-de-upsert",
3729 : : "insert or ignore into " BUILDIDS "_f_de "
3730 : : "(buildid, debuginfo_p, executable_p, file, mtime) "
3731 : : "values ((select id from " BUILDIDS "_buildids where hex = ?),"
3732 [ + - + - : 512 : " ?,?,?,?);");
+ - + - -
- ]
3733 [ + - ]: 256 : sqlite_ps ps_f_upsert_s (db, "file-s-upsert",
3734 : : "insert or ignore into " BUILDIDS "_f_s "
3735 : : "(buildid, artifactsrc, file, mtime) "
3736 : : "values ((select id from " BUILDIDS "_buildids where hex = ?),"
3737 [ + - + - : 512 : " ?,?,?);");
+ - + - -
- ]
3738 [ + - ]: 256 : sqlite_ps ps_f_query (db, "file-negativehit-find",
3739 : : "select 1 from " BUILDIDS "_file_mtime_scanned where sourcetype = 'F' "
3740 [ + - + - : 512 : "and file = ? and mtime = ?;");
+ - + - -
- ]
3741 [ + - ]: 256 : sqlite_ps ps_f_scan_done (db, "file-scanned",
3742 : : "insert or ignore into " BUILDIDS "_file_mtime_scanned (sourcetype, file, mtime, size)"
3743 [ + - + - : 512 : "values ('F', ?,?,?);");
+ - + - -
- ]
3744 : :
3745 : : // and now for the _r_ set
3746 [ + - + - : 512 : sqlite_ps ps_r_upsert_buildids (db, "rpm-buildid-intern", "insert or ignore into " BUILDIDS "_buildids VALUES (NULL, ?);");
+ - + - -
- ]
3747 [ + - + - : 512 : sqlite_ps ps_r_upsert_fileparts (db, "rpm-fileparts-intern", "insert or ignore into " BUILDIDS "_fileparts VALUES (NULL, ?);");
+ - + - -
- ]
3748 [ + - ]: 256 : sqlite_ps ps_r_upsert_file (db, "rpm-file-intern", "insert or ignore into " BUILDIDS "_files VALUES (NULL, \n"
3749 : : "(select id from " BUILDIDS "_fileparts where name = ?),\n"
3750 [ + - + - : 512 : "(select id from " BUILDIDS "_fileparts where name = ?));");
+ - + - -
- ]
3751 [ + - ]: 256 : sqlite_ps ps_r_lookup_file (db, "rpm-file-lookup",
3752 : : "select f.id\n"
3753 : : " from " BUILDIDS "_files f, " BUILDIDS "_fileparts p1, " BUILDIDS "_fileparts p2 \n"
3754 [ + - + - : 512 : " where f.dirname = p1.id and f.basename = p2.id and p1.name = ? and p2.name = ?;\n");
+ - + - -
- ]
3755 [ + - ]: 256 : sqlite_ps ps_r_upsert_de (db, "rpm-de-insert",
3756 : : "insert or ignore into " BUILDIDS "_r_de (buildid, debuginfo_p, executable_p, file, mtime, content) values ("
3757 [ + - + - : 512 : "(select id from " BUILDIDS "_buildids where hex = ?), ?, ?, ?, ?, ?);");
+ - + - -
- ]
3758 [ + - ]: 256 : sqlite_ps ps_r_upsert_sref (db, "rpm-sref-insert",
3759 : : "insert or ignore into " BUILDIDS "_r_sref (buildid, artifactsrc) values ("
3760 : : "(select id from " BUILDIDS "_buildids where hex = ?), "
3761 [ + - + - : 512 : "?);");
+ - + - -
- ]
3762 [ + - ]: 256 : sqlite_ps ps_r_upsert_sdef (db, "rpm-sdef-insert",
3763 : : "insert or ignore into " BUILDIDS "_r_sdef (file, mtime, content) values ("
3764 [ + - + - : 512 : "?, ?, ?);");
+ - + - -
- ]
3765 [ + - ]: 256 : sqlite_ps ps_r_query (db, "rpm-negativehit-query",
3766 : : "select 1 from " BUILDIDS "_file_mtime_scanned where "
3767 [ + - + - : 512 : "sourcetype = 'R' and file = ? and mtime = ?;");
+ - + - -
- ]
3768 [ + - ]: 256 : sqlite_ps ps_r_scan_done (db, "rpm-scanned",
3769 : : "insert or ignore into " BUILDIDS "_file_mtime_scanned (sourcetype, file, mtime, size)"
3770 [ + - + - : 512 : "values ('R', ?, ?, ?);");
+ - + - -
- ]
3771 : :
3772 : :
3773 : 256 : unsigned fts_cached = 0, fts_executable = 0, fts_debuginfo = 0, fts_sourcefiles = 0;
3774 : 256 : unsigned fts_sref = 0, fts_sdef = 0;
3775 : :
3776 [ + - + - : 512 : add_metric("thread_count", "role", "scan", 1);
+ - + - -
+ - + - -
- - ]
3777 [ + - + - : 512 : add_metric("thread_busy", "role", "scan", 1);
+ - + - -
+ - + - -
- - ]
3778 [ + + ]: 1452 : while (! interrupted)
3779 : : {
3780 [ + - ]: 1196 : scan_payload p;
3781 : :
3782 [ + - + - : 2392 : add_metric("thread_busy", "role", "scan", -1);
+ - + - -
+ - + - -
- - ]
3783 : : // NB: threads may be blocked within either of these two waiting
3784 : : // states, if the work queue happens to run dry. That's OK.
3785 [ + - + - ]: 1196 : if (scan_barrier) scan_barrier->count();
3786 [ + - ]: 1196 : bool gotone = scanq.wait_front(p);
3787 [ + - + - : 2392 : add_metric("thread_busy", "role", "scan", 1);
+ - + - -
+ - + - -
- - ]
3788 : :
3789 [ + + - + ]: 1196 : if (! gotone) continue; // go back to waiting
3790 : :
3791 : 940 : try
3792 : : {
3793 : 940 : bool scan_archive = false;
3794 [ + + ]: 2240 : for (auto&& arch : scan_archives)
3795 [ + + ]: 1300 : if (string_endswith(p.first, arch.first))
3796 : 702 : scan_archive = true;
3797 : :
3798 [ + + ]: 940 : if (scan_archive)
3799 [ + - ]: 702 : scan_archive_file (p.first, p.second,
3800 : : ps_r_upsert_buildids,
3801 : : ps_r_upsert_fileparts,
3802 : : ps_r_upsert_file,
3803 : : ps_r_lookup_file,
3804 : : ps_r_upsert_de,
3805 : : ps_r_upsert_sref,
3806 : : ps_r_upsert_sdef,
3807 : : ps_r_query,
3808 : : ps_r_scan_done,
3809 : : fts_cached,
3810 : : fts_executable,
3811 : : fts_debuginfo,
3812 : : fts_sref,
3813 : : fts_sdef);
3814 : :
3815 [ + + ]: 940 : if (scan_files) // NB: maybe "else if" ?
3816 [ + - ]: 728 : scan_source_file (p.first, p.second,
3817 : : ps_f_upsert_buildids,
3818 : : ps_f_upsert_fileparts,
3819 : : ps_f_upsert_file,
3820 : : ps_f_lookup_file,
3821 : : ps_f_upsert_de,
3822 : : ps_f_upsert_s,
3823 : : ps_f_query,
3824 : : ps_f_scan_done,
3825 : : fts_cached, fts_executable, fts_debuginfo, fts_sourcefiles);
3826 : : }
3827 [ - - ]: 0 : catch (const reportable_exception& e)
3828 : : {
3829 [ - - ]: 0 : e.report(cerr);
3830 : 0 : }
3831 : :
3832 [ + - ]: 940 : scanq.done_front(); // let idlers run
3833 : :
3834 : 940 : if (fts_cached || fts_executable || fts_debuginfo || fts_sourcefiles || fts_sref || fts_sdef)
3835 : : {} // NB: not just if a successful scan - we might have encountered -ENOSPC & failed
3836 [ + - + - ]: 940 : (void) statfs_free_enough_p(db_path, "database"); // report sqlite filesystem size
3837 [ + - + - ]: 940 : (void) statfs_free_enough_p(tmpdir, "tmpdir"); // this too, in case of fdcache/tmpfile usage
3838 : :
3839 : : // finished a scanning step -- not a "loop", because we just
3840 : : // consume the traversal loop's work, whenever
3841 [ + - + - : 1880 : inc_metric("thread_work_total","role","scan");
+ - + - -
+ - + + -
- - - - -
- ]
3842 : 1196 : }
3843 : :
3844 [ + - + - : 512 : add_metric("thread_busy", "role", "scan", -1);
+ - + - -
+ - + - -
- - ]
3845 : 256 : }
3846 : :
3847 : :
3848 : : // Use this function as the thread entry point, so it can catch our
3849 : : // fleet of exceptions (incl. the sqlite_ps ctors) and report.
3850 : : static void*
3851 : 256 : thread_main_scanner (void* arg)
3852 : : {
3853 : 256 : (void) arg;
3854 [ + + ]: 768 : while (! interrupted)
3855 : 256 : try
3856 : : {
3857 [ + - ]: 256 : scan();
3858 : : }
3859 [ - - ]: 0 : catch (const reportable_exception& e)
3860 : : {
3861 [ - - ]: 0 : e.report(cerr);
3862 : 0 : }
3863 : 256 : return 0;
3864 : : }
3865 : :
3866 : :
3867 : :
3868 : : // The thread that traverses all the source_paths and enqueues all the
3869 : : // matching files into the file/archive scan queue.
3870 : : static void
3871 : 119 : scan_source_paths()
3872 : : {
3873 : : // NB: fedora 31 glibc/fts(3) crashes inside fts_read() on empty
3874 : : // path list.
3875 [ + + ]: 119 : if (source_paths.empty())
3876 : 2 : return;
3877 : :
3878 : : // Turn the source_paths into an fts(3)-compatible char**. Since
3879 : : // source_paths[] does not change after argv processing, the
3880 : : // c_str()'s are safe to keep around awile.
3881 : 117 : vector<const char *> sps;
3882 [ + + ]: 316 : for (auto&& sp: source_paths)
3883 [ + - ]: 199 : sps.push_back(sp.c_str());
3884 [ + - - - ]: 117 : sps.push_back(NULL);
3885 : :
3886 [ + + + - ]: 220 : FTS *fts = fts_open ((char * const *)sps.data(),
3887 : : (traverse_logical ? FTS_LOGICAL : FTS_PHYSICAL|FTS_XDEV)
3888 : : | FTS_NOCHDIR /* multithreaded */,
3889 : : NULL);
3890 [ - + ]: 117 : if (fts == NULL)
3891 [ # # # # ]: 0 : throw libc_exception(errno, "cannot fts_open");
3892 : 117 : defer_dtor<FTS*,int> fts_cleanup (fts, fts_close);
3893 : :
3894 : 117 : struct timespec ts_start, ts_end;
3895 : 117 : clock_gettime (CLOCK_MONOTONIC, &ts_start);
3896 : 117 : unsigned fts_scanned = 0, fts_regex = 0;
3897 : :
3898 : 117 : FTSENT *f;
3899 [ + - + + ]: 2119 : while ((f = fts_read (fts)) != NULL)
3900 : : {
3901 [ + - ]: 1885 : if (interrupted) break;
3902 : :
3903 [ - + ]: 1885 : if (sigusr2 != forced_groom_count) // stop early if groom triggered
3904 : : {
3905 [ # # ]: 0 : scanq.clear(); // clear previously issued work for scanner threads
3906 : : break;
3907 : : }
3908 : :
3909 : 1885 : fts_scanned ++;
3910 : :
3911 [ + - ]: 1885 : if (verbose > 2)
3912 [ + - + - : 3770 : obatched(clog) << "fts traversing " << f->fts_path << endl;
+ - ]
3913 : :
3914 [ + + + + : 1885 : switch (f->fts_info)
+ ]
3915 : : {
3916 : 1028 : case FTS_F:
3917 : 1028 : {
3918 : : /* Found a file. Convert it to an absolute path, so
3919 : : the buildid database does not have relative path
3920 : : names that are unresolvable from a subsequent run
3921 : : in a different cwd. */
3922 [ + - ]: 1028 : char *rp = realpath(f->fts_path, NULL);
3923 [ - + ]: 1028 : if (rp == NULL)
3924 : 0 : continue; // ignore dangling symlink or such
3925 [ + - ]: 1028 : string rps = string(rp);
3926 : 1028 : free (rp);
3927 : :
3928 [ + - ]: 1028 : bool ri = !regexec (&file_include_regex, rps.c_str(), 0, 0, 0);
3929 [ + - ]: 1028 : bool rx = !regexec (&file_exclude_regex, rps.c_str(), 0, 0, 0);
3930 [ + + ]: 1028 : if (!ri || rx)
3931 : : {
3932 [ + - ]: 88 : if (verbose > 3)
3933 [ + - ]: 176 : obatched(clog) << "fts skipped by regex "
3934 [ + + + - : 96 : << (!ri ? "I" : "") << (rx ? "X" : "") << endl;
+ + + - +
- ]
3935 : 88 : fts_regex ++;
3936 [ + + ]: 88 : if (!ri)
3937 [ + - + - : 8 : inc_metric("traversed_total","type","file-skipped-I");
+ - + - -
+ - + - -
- - ]
3938 [ + + ]: 88 : if (rx)
3939 [ + - + - : 168 : inc_metric("traversed_total","type","file-skipped-X");
+ - + - -
+ - + - -
- - ]
3940 : : }
3941 : : else
3942 : : {
3943 [ + - + - ]: 940 : scanq.push_back (make_pair(rps, *f->fts_statp));
3944 [ + - + - : 1880 : inc_metric("traversed_total","type","file");
+ - + - -
+ - + - -
- - - - ]
3945 : : }
3946 : 0 : }
3947 : 1028 : break;
3948 : :
3949 : 5 : case FTS_ERR:
3950 : 5 : case FTS_NS:
3951 : : // report on some types of errors because they may reflect fixable misconfiguration
3952 : 5 : {
3953 [ + - + - : 10 : auto x = libc_exception(f->fts_errno, string("fts traversal ") + string(f->fts_path));
+ - + - -
+ - + -
- ]
3954 [ + - ]: 5 : x.report(cerr);
3955 : 0 : }
3956 [ + - + - : 10 : inc_metric("traversed_total","type","error");
+ - + - -
+ - + - -
- - ]
3957 : 5 : break;
3958 : :
3959 : 12 : case FTS_SL: // ignore, but count because debuginfod -L would traverse these
3960 [ + - + - : 24 : inc_metric("traversed_total","type","symlink");
+ - + - -
+ - + - -
- - ]
3961 : 12 : break;
3962 : :
3963 : 420 : case FTS_D: // ignore
3964 [ + - + - : 840 : inc_metric("traversed_total","type","directory");
+ - + - -
+ - + - -
- - ]
3965 : 420 : break;
3966 : :
3967 : 420 : default: // ignore
3968 [ + - + - : 840 : inc_metric("traversed_total","type","other");
+ - + - -
+ - + - -
- - ]
3969 : 420 : break;
3970 : : }
3971 : : }
3972 : 117 : clock_gettime (CLOCK_MONOTONIC, &ts_end);
3973 : 117 : double deltas = (ts_end.tv_sec - ts_start.tv_sec) + (ts_end.tv_nsec - ts_start.tv_nsec)/1.e9;
3974 : :
3975 [ + - + - : 351 : obatched(clog) << "fts traversed source paths in " << deltas << "s, scanned=" << fts_scanned
+ - + - ]
3976 [ + - + - : 117 : << ", regex-skipped=" << fts_regex << endl;
+ - ]
3977 [ + - ]: 234 : }
3978 : :
3979 : :
3980 : : static void*
3981 : 64 : thread_main_fts_source_paths (void* arg)
3982 : : {
3983 : 64 : (void) arg; // ignore; we operate on global data
3984 : :
3985 [ + - + - : 128 : set_metric("thread_tid", "role","traverse", tid());
+ - - + -
+ - - -
- ]
3986 [ + - + - : 128 : add_metric("thread_count", "role", "traverse", 1);
+ - - + -
+ - - -
- ]
3987 : :
3988 : 64 : time_t last_rescan = 0;
3989 : :
3990 [ + - ]: 303 : while (! interrupted)
3991 : : {
3992 : 303 : sleep (1);
3993 : 303 : scanq.wait_idle(); // don't start a new traversal while scanners haven't finished the job
3994 : 303 : scanq.done_idle(); // release the hounds
3995 [ + + ]: 303 : if (interrupted) break;
3996 : :
3997 : 239 : time_t now = time(NULL);
3998 : 239 : bool rescan_now = false;
3999 [ + + ]: 239 : if (last_rescan == 0) // at least one initial rescan is documented even for -t0
4000 : 63 : rescan_now = true;
4001 [ + + + + ]: 239 : if (rescan_s > 0 && (long)now > (long)(last_rescan + rescan_s))
4002 : 239 : rescan_now = true;
4003 [ + + ]: 239 : if (sigusr1 != forced_rescan_count)
4004 : : {
4005 : 62 : forced_rescan_count = sigusr1;
4006 : 62 : rescan_now = true;
4007 : : }
4008 [ + + ]: 239 : if (rescan_now)
4009 : : {
4010 [ + - + - : 238 : set_metric("thread_busy", "role","traverse", 1);
+ - - + -
+ - - -
- ]
4011 : 119 : try
4012 : : {
4013 [ + - ]: 119 : scan_source_paths();
4014 : : }
4015 [ - - ]: 0 : catch (const reportable_exception& e)
4016 : : {
4017 [ - - ]: 0 : e.report(cerr);
4018 : 0 : }
4019 : 119 : last_rescan = time(NULL); // NB: now was before scanning
4020 : : // finished a traversal loop
4021 [ + - + - : 238 : inc_metric("thread_work_total", "role","traverse");
+ - - + -
+ - - -
- ]
4022 [ + - + - : 238 : set_metric("thread_busy", "role","traverse", 0);
+ - - + -
+ - - -
- ]
4023 : : }
4024 : : }
4025 : :
4026 : 64 : return 0;
4027 : : }
4028 : :
4029 : :
4030 : :
4031 : : ////////////////////////////////////////////////////////////////////////
4032 : :
4033 : : static void
4034 : 71 : database_stats_report()
4035 : : {
4036 : 71 : sqlite_ps ps_query (db, "database-overview",
4037 [ + - + - : 142 : "select label,quantity from " BUILDIDS "_stats");
+ - - - ]
4038 : :
4039 [ + - + - ]: 142 : obatched(clog) << "database record counts:" << endl;
4040 : 1633 : while (1)
4041 : : {
4042 [ + - ]: 852 : if (interrupted) break;
4043 [ + - ]: 852 : if (sigusr1 != forced_rescan_count) // stop early if scan triggered
4044 : : break;
4045 : :
4046 [ + - ]: 852 : int rc = ps_query.step();
4047 [ + + ]: 852 : if (rc == SQLITE_DONE) break;
4048 [ - + ]: 781 : if (rc != SQLITE_ROW)
4049 [ # # # # ]: 0 : throw sqlite_exception(rc, "step");
4050 : :
4051 [ + - ]: 781 : obatched(clog)
4052 [ + - - + : 781 : << ((const char*) sqlite3_column_text(ps_query, 0) ?: (const char*) "NULL")
+ - ]
4053 : : << " "
4054 [ + - + - : 1562 : << (sqlite3_column_text(ps_query, 1) ?: (const unsigned char*) "NULL")
- + + - ]
4055 : 781 : << endl;
4056 : :
4057 [ + - + - : 1562 : set_metric("groom", "statistic",
- + + - +
- + - + -
- + + + -
- - - ]
4058 [ + - ]: 781 : ((const char*) sqlite3_column_text(ps_query, 0) ?: (const char*) "NULL"),
4059 : : (sqlite3_column_double(ps_query, 1)));
4060 : 781 : }
4061 : 71 : }
4062 : :
4063 : :
4064 : : // Do a round of database grooming that might take many minutes to run.
4065 : 71 : void groom()
4066 : : {
4067 [ + - ]: 142 : obatched(clog) << "grooming database" << endl;
4068 : :
4069 : 71 : struct timespec ts_start, ts_end;
4070 : 71 : clock_gettime (CLOCK_MONOTONIC, &ts_start);
4071 : :
4072 : : // scan for files that have disappeared
4073 : 71 : sqlite_ps files (db, "check old files",
4074 : : "select distinct s.mtime, s.file, f.name from "
4075 : : BUILDIDS "_file_mtime_scanned s, " BUILDIDS "_files_v f "
4076 [ + - + - : 142 : "where f.id = s.file");
+ - - - ]
4077 : : // NB: Because _ftime_mtime_scanned can contain both F and
4078 : : // R records for the same file, this query would return duplicates if the
4079 : : // DISTINCT qualifier were not there.
4080 [ + - ]: 71 : files.reset();
4081 : :
4082 : : // DECISION TIME - we enumerate stale fileids/mtimes
4083 [ + - ]: 71 : deque<pair<int64_t,int64_t> > stale_fileid_mtime;
4084 : :
4085 : 71 : time_t time_start = time(NULL);
4086 : 303 : while(1)
4087 : : {
4088 : : // PR28514: limit grooming iteration to O(rescan time), to avoid
4089 : : // slow filesystem tests over many files locking out rescans for
4090 : : // too long.
4091 [ + + - + ]: 187 : if (rescan_s > 0 && (long)time(NULL) > (long)(time_start + rescan_s))
4092 : : {
4093 [ # # # # : 0 : inc_metric("groomed_total", "decision", "aborted");
# # # # #
# # # # #
# # ]
4094 : 0 : break;
4095 : : }
4096 : :
4097 [ + - ]: 187 : if (interrupted) break;
4098 : :
4099 [ + - ]: 187 : int rc = files.step();
4100 [ + + ]: 187 : if (rc != SQLITE_ROW)
4101 : : break;
4102 : :
4103 [ + - ]: 116 : int64_t mtime = sqlite3_column_int64 (files, 0);
4104 [ + - ]: 116 : int64_t fileid = sqlite3_column_int64 (files, 1);
4105 [ + - - + ]: 116 : const char* filename = ((const char*) sqlite3_column_text (files, 2) ?: "");
4106 : 116 : struct stat s;
4107 : 116 : bool regex_file_drop = 0;
4108 : :
4109 [ + + ]: 116 : if (regex_groom)
4110 : : {
4111 [ + - ]: 16 : bool reg_include = !regexec (&file_include_regex, filename, 0, 0, 0);
4112 [ + - ]: 16 : bool reg_exclude = !regexec (&file_exclude_regex, filename, 0, 0, 0);
4113 : 16 : regex_file_drop = !reg_include || reg_exclude; // match logic of scan_source_paths
4114 : : }
4115 : :
4116 : 116 : rc = stat(filename, &s);
4117 [ + + - + ]: 116 : if ( regex_file_drop || rc < 0 || (mtime != (int64_t) s.st_mtime) )
4118 : : {
4119 [ + - ]: 24 : if (verbose > 2)
4120 [ + - + - : 48 : obatched(clog) << "groom: stale file=" << filename << " mtime=" << mtime << endl;
+ - + - +
- ]
4121 [ + - ]: 24 : stale_fileid_mtime.push_back(make_pair(fileid,mtime));
4122 [ + - + - : 48 : inc_metric("groomed_total", "decision", "stale");
+ - + - -
+ - + - -
- - ]
4123 [ + - + - : 48 : set_metric("thread_work_pending","role","groom", stale_fileid_mtime.size());
+ - + - -
+ - + - -
- - ]
4124 : : }
4125 : : else
4126 [ + - + - : 184 : inc_metric("groomed_total", "decision", "fresh");
+ - + - -
+ - + - -
- - ]
4127 : :
4128 [ + - ]: 116 : if (sigusr1 != forced_rescan_count) // stop early if scan triggered
4129 : : break;
4130 : 116 : }
4131 [ + - ]: 71 : files.reset();
4132 : :
4133 : : // ACTION TIME
4134 : :
4135 : : // Now that we know which file/mtime tuples are stale, actually do
4136 : : // the deletion from the database. Doing this during the SELECT
4137 : : // iteration above results in undefined behaviour in sqlite, as per
4138 : : // https://www.sqlite.org/isolation.html
4139 : :
4140 : : // We could shuffle stale_fileid_mtime[] here. It'd let aborted
4141 : : // sequences of nuke operations resume at random locations, instead
4142 : : // of just starting over. But it doesn't matter much either way,
4143 : : // as long as we make progress.
4144 : :
4145 [ + - + - : 142 : sqlite_ps files_del_f_de (db, "nuke f_de", "delete from " BUILDIDS "_f_de where file = ? and mtime = ?");
+ - + - -
- ]
4146 [ + - + - : 142 : sqlite_ps files_del_r_de (db, "nuke r_de", "delete from " BUILDIDS "_r_de where file = ? and mtime = ?");
+ - + - -
- ]
4147 [ + - ]: 71 : sqlite_ps files_del_scan (db, "nuke f_m_s", "delete from " BUILDIDS "_file_mtime_scanned "
4148 [ + - + - : 142 : "where file = ? and mtime = ?");
+ - + - -
- ]
4149 : :
4150 [ + + ]: 95 : while (! stale_fileid_mtime.empty())
4151 : : {
4152 : 24 : auto stale = stale_fileid_mtime.front();
4153 : 24 : stale_fileid_mtime.pop_front();
4154 [ + - + - : 48 : set_metric("thread_work_pending","role","groom", stale_fileid_mtime.size());
+ - + - -
+ - + - -
- - ]
4155 : :
4156 : : // PR28514: limit grooming iteration to O(rescan time), to avoid
4157 : : // slow nuke_* queries over many files locking out rescans for too
4158 : : // long. We iterate over the files in random() sequence to avoid
4159 : : // partial checks going over the same set.
4160 [ - + - - ]: 24 : if (rescan_s > 0 && (long)time(NULL) > (long)(time_start + rescan_s))
4161 : : {
4162 [ # # # # : 0 : inc_metric("groomed_total", "action", "aborted");
# # # # #
# # # # #
# # ]
4163 : 0 : break;
4164 : : }
4165 : :
4166 [ + - ]: 24 : if (interrupted) break;
4167 : :
4168 : 24 : int64_t fileid = stale.first;
4169 : 24 : int64_t mtime = stale.second;
4170 [ + - + - : 24 : files_del_f_de.reset().bind(1,fileid).bind(2,mtime).step_ok_done();
+ - + - ]
4171 [ + - + - : 24 : files_del_r_de.reset().bind(1,fileid).bind(2,mtime).step_ok_done();
+ - + - ]
4172 [ + - + - : 24 : files_del_scan.reset().bind(1,fileid).bind(2,mtime).step_ok_done();
+ - + - ]
4173 [ + - + - : 48 : inc_metric("groomed_total", "action", "cleaned");
+ - + - -
+ - + - -
- - ]
4174 : :
4175 [ + - ]: 24 : if (sigusr1 != forced_rescan_count) // stop early if scan triggered
4176 : : break;
4177 : : }
4178 : 71 : stale_fileid_mtime.clear(); // no need for this any longer
4179 [ + - + - : 142 : set_metric("thread_work_pending","role","groom", stale_fileid_mtime.size());
+ - + - -
+ - + - -
- - ]
4180 : :
4181 : : // delete buildids with no references in _r_de or _f_de tables;
4182 : : // cascades to _r_sref & _f_s records
4183 [ + - ]: 71 : sqlite_ps buildids_del (db, "nuke orphan buildids",
4184 : : "delete from " BUILDIDS "_buildids "
4185 : : "where not exists (select 1 from " BUILDIDS "_f_de d where " BUILDIDS "_buildids.id = d.buildid) "
4186 [ + - + - : 142 : "and not exists (select 1 from " BUILDIDS "_r_de d where " BUILDIDS "_buildids.id = d.buildid)");
+ - + - -
- ]
4187 [ + - + - ]: 71 : buildids_del.reset().step_ok_done();
4188 : :
4189 [ - + ]: 71 : if (interrupted) return;
4190 : :
4191 : : // NB: "vacuum" is too heavy for even daily runs: it rewrites the entire db, so is done as maxigroom -G
4192 [ + - + - : 142 : sqlite_ps g1 (db, "incremental vacuum", "pragma incremental_vacuum");
+ - + - -
- ]
4193 [ + - + - ]: 71 : g1.reset().step_ok_done();
4194 [ + - + - : 142 : sqlite_ps g2 (db, "optimize", "pragma optimize");
+ - - + -
- ]
4195 [ + - + - ]: 71 : g2.reset().step_ok_done();
4196 [ + - + - : 142 : sqlite_ps g3 (db, "wal checkpoint", "pragma wal_checkpoint=truncate");
+ - + - -
- ]
4197 [ + - + - ]: 71 : g3.reset().step_ok_done();
4198 : :
4199 [ + - ]: 71 : database_stats_report();
4200 : :
4201 [ + - + - ]: 71 : (void) statfs_free_enough_p(db_path, "database"); // report sqlite filesystem size
4202 : :
4203 [ + - ]: 71 : sqlite3_db_release_memory(db); // shrink the process if possible
4204 [ + - ]: 71 : sqlite3_db_release_memory(dbq); // ... for both connections
4205 [ + - ]: 71 : debuginfod_pool_groom(); // and release any debuginfod_client objects we've been holding onto
4206 : :
4207 [ + - ]: 71 : fdcache.limit(0,0,0,0); // release the fdcache contents
4208 [ + - ]: 71 : fdcache.limit(fdcache_fds, fdcache_mbs, fdcache_prefetch_fds, fdcache_prefetch_mbs); // restore status quo parameters
4209 : :
4210 : 71 : clock_gettime (CLOCK_MONOTONIC, &ts_end);
4211 : 71 : double deltas = (ts_end.tv_sec - ts_start.tv_sec) + (ts_end.tv_nsec - ts_start.tv_nsec)/1.e9;
4212 : :
4213 [ + - + - : 142 : obatched(clog) << "groomed database in " << deltas << "s" << endl;
+ - + - ]
4214 : 71 : }
4215 : :
4216 : :
4217 : : static void*
4218 : 70 : thread_main_groom (void* /*arg*/)
4219 : : {
4220 [ + - + - : 140 : set_metric("thread_tid", "role", "groom", tid());
+ - - + -
+ - - -
- ]
4221 [ + - + - : 140 : add_metric("thread_count", "role", "groom", 1);
+ - - + -
+ - - -
- ]
4222 : :
4223 : 70 : time_t last_groom = 0;
4224 : :
4225 : 550 : while (1)
4226 : : {
4227 : 310 : sleep (1);
4228 : 310 : scanq.wait_idle(); // PR25394: block scanners during grooming!
4229 [ + + ]: 310 : if (interrupted) break;
4230 : :
4231 : 240 : time_t now = time(NULL);
4232 : 240 : bool groom_now = false;
4233 [ + + ]: 240 : if (last_groom == 0) // at least one initial groom is documented even for -g0
4234 : 65 : groom_now = true;
4235 [ + + + + ]: 240 : if (groom_s > 0 && (long)now > (long)(last_groom + groom_s))
4236 : 240 : groom_now = true;
4237 [ + + ]: 240 : if (sigusr2 != forced_groom_count)
4238 : : {
4239 : 6 : forced_groom_count = sigusr2;
4240 : 6 : groom_now = true;
4241 : : }
4242 [ + + ]: 240 : if (groom_now)
4243 : : {
4244 [ + - + - : 142 : set_metric("thread_busy", "role", "groom", 1);
+ - - + -
+ - - -
- ]
4245 : 71 : try
4246 : : {
4247 [ + - ]: 71 : groom ();
4248 : : }
4249 [ - - ]: 0 : catch (const sqlite_exception& e)
4250 : : {
4251 [ - - - - : 0 : obatched(cerr) << e.message << endl;
- - ]
4252 : 0 : }
4253 : 71 : last_groom = time(NULL); // NB: now was before grooming
4254 : : // finished a grooming loop
4255 [ + - + - : 142 : inc_metric("thread_work_total", "role", "groom");
+ - - + -
+ - - -
- ]
4256 [ + - + - : 142 : set_metric("thread_busy", "role", "groom", 0);
+ - - + -
+ - - -
- ]
4257 : : }
4258 : :
4259 : 240 : scanq.done_idle();
4260 : 240 : }
4261 : :
4262 : 70 : return 0;
4263 : : }
4264 : :
4265 : :
4266 : : ////////////////////////////////////////////////////////////////////////
4267 : :
4268 : :
4269 : : static void
4270 : 72 : signal_handler (int /* sig */)
4271 : : {
4272 : 72 : interrupted ++;
4273 : :
4274 [ + + ]: 72 : if (db)
4275 : 70 : sqlite3_interrupt (db);
4276 [ + - ]: 72 : if (dbq)
4277 : 72 : sqlite3_interrupt (dbq);
4278 : :
4279 : : // NB: don't do anything else in here
4280 : 72 : }
4281 : :
4282 : : static void
4283 : 62 : sigusr1_handler (int /* sig */)
4284 : : {
4285 : 62 : sigusr1 ++;
4286 : : // NB: don't do anything else in here
4287 : 62 : }
4288 : :
4289 : : static void
4290 : 6 : sigusr2_handler (int /* sig */)
4291 : : {
4292 : 6 : sigusr2 ++;
4293 : : // NB: don't do anything else in here
4294 : 6 : }
4295 : :
4296 : :
4297 : : static void // error logging callback from libmicrohttpd internals
4298 : 0 : error_cb (void *arg, const char *fmt, va_list ap)
4299 : : {
4300 : 0 : (void) arg;
4301 [ # # # # : 0 : inc_metric("error_count","libmicrohttpd",fmt);
# # # # #
# # # #
# ]
4302 : 0 : char errmsg[512];
4303 : 0 : (void) vsnprintf (errmsg, sizeof(errmsg), fmt, ap); // ok if slightly truncated
4304 [ # # ]: 0 : obatched(cerr) << "libmicrohttpd error: " << errmsg; // MHD_DLOG calls already include \n
4305 : 0 : }
4306 : :
4307 : :
4308 : : // A user-defined sqlite function, to score the sharedness of the
4309 : : // prefix of two strings. This is used to compare candidate debuginfo
4310 : : // / source-rpm names, so that the closest match
4311 : : // (directory-topology-wise closest) is found. This is important in
4312 : : // case the same sref (source file name) is in many -debuginfo or
4313 : : // -debugsource RPMs, such as when multiple versions/releases of the
4314 : : // same package are in the database.
4315 : :
4316 : 296 : static void sqlite3_sharedprefix_fn (sqlite3_context* c, int argc, sqlite3_value** argv)
4317 : : {
4318 [ - + ]: 296 : if (argc != 2)
4319 : 0 : sqlite3_result_error(c, "expect 2 string arguments", -1);
4320 [ + - + + ]: 592 : else if ((sqlite3_value_type(argv[0]) != SQLITE_TEXT) ||
4321 : 296 : (sqlite3_value_type(argv[1]) != SQLITE_TEXT))
4322 : 6 : sqlite3_result_null(c);
4323 : : else
4324 : : {
4325 : 290 : const unsigned char* a = sqlite3_value_text (argv[0]);
4326 : 290 : const unsigned char* b = sqlite3_value_text (argv[1]);
4327 : 290 : int i = 0;
4328 [ + + + - : 35878 : while (*a != '\0' && *b != '\0' && *a++ == *b++)
+ + + + ]
4329 : 35048 : i++;
4330 : 290 : sqlite3_result_int (c, i);
4331 : : }
4332 : 296 : }
4333 : :
4334 : :
4335 : : static unsigned
4336 : 140 : default_concurrency() // guaranteed >= 1
4337 : : {
4338 : : // Prior to PR29975 & PR29976, we'd just use this:
4339 : 140 : unsigned sth = std::thread::hardware_concurrency();
4340 : : // ... but on many-CPU boxes, admins or distros may throttle
4341 : : // resources in such a way that debuginfod would mysteriously fail.
4342 : : // So we reduce the defaults:
4343 : :
4344 : 140 : unsigned aff = 0;
4345 : : #ifdef HAVE_SCHED_GETAFFINITY
4346 : 140 : {
4347 : 140 : int ret;
4348 : 140 : cpu_set_t mask;
4349 : 140 : CPU_ZERO(&mask);
4350 : 140 : ret = sched_getaffinity(0, sizeof(mask), &mask);
4351 [ + - ]: 140 : if (ret == 0)
4352 : 140 : aff = CPU_COUNT(&mask);
4353 : : }
4354 : : #endif
4355 : :
4356 : 140 : unsigned fn = 0;
4357 : : #ifdef HAVE_GETRLIMIT
4358 : 140 : {
4359 : 140 : struct rlimit rlim;
4360 : 140 : int rc = getrlimit(RLIMIT_NOFILE, &rlim);
4361 [ + - ]: 140 : if (rc == 0)
4362 [ + - ]: 280 : fn = max((rlim_t)1, (rlim.rlim_cur - 100) / 4);
4363 : : // at least 2 fds are used by each listener thread etc.
4364 : : // plus a bunch to account for shared libraries and such
4365 : : }
4366 : : #endif
4367 : :
4368 [ - + - + : 140 : unsigned d = min(max(sth, 1U),
- + ]
4369 [ - + ]: 140 : min(max(aff, 1U),
4370 [ - + ]: 140 : max(fn, 1U)));
4371 : 140 : return d;
4372 : : }
4373 : :
4374 : :
4375 : : // 30879: Something to help out in case of an uncaught exception.
4376 : 0 : void my_terminate_handler()
4377 : : {
4378 : : #if defined(__GLIBC__)
4379 : 0 : void *array[40];
4380 : 0 : int size = backtrace (array, 40);
4381 : 0 : backtrace_symbols_fd (array, size, STDERR_FILENO);
4382 : : #endif
4383 : : #if defined(__GLIBCXX__) || defined(__GLIBCPP__)
4384 : 0 : __gnu_cxx::__verbose_terminate_handler();
4385 : : #endif
4386 : 0 : abort();
4387 : : }
4388 : :
4389 : :
4390 : : int
4391 : 72 : main (int argc, char *argv[])
4392 : : {
4393 : 72 : (void) setlocale (LC_ALL, "");
4394 : 72 : (void) bindtextdomain (PACKAGE_TARNAME, LOCALEDIR);
4395 : 72 : (void) textdomain (PACKAGE_TARNAME);
4396 : :
4397 : 72 : std::set_terminate(& my_terminate_handler);
4398 : :
4399 : : /* Tell the library which version we are expecting. */
4400 : 72 : elf_version (EV_CURRENT);
4401 : :
4402 [ + - - + ]: 144 : tmpdir = string(getenv("TMPDIR") ?: "/tmp");
4403 : :
4404 : : /* Set computed default values. */
4405 [ - + + - : 72 : db_path = string(getenv("HOME") ?: "/") + string("/.debuginfod.sqlite"); /* XDG? */
+ - - + -
+ + - -
- ]
4406 : 72 : int rc = regcomp (& file_include_regex, ".*", REG_EXTENDED|REG_NOSUB); // match everything
4407 [ - + ]: 72 : if (rc != 0)
4408 : 0 : error (EXIT_FAILURE, 0, "regcomp failure: %d", rc);
4409 : 72 : rc = regcomp (& file_exclude_regex, "^$", REG_EXTENDED|REG_NOSUB); // match nothing
4410 [ - + ]: 72 : if (rc != 0)
4411 : 0 : error (EXIT_FAILURE, 0, "regcomp failure: %d", rc);
4412 : :
4413 : : // default parameters for fdcache are computed from system stats
4414 : 72 : struct statfs sfs;
4415 : 72 : rc = statfs(tmpdir.c_str(), &sfs);
4416 [ - + ]: 72 : if (rc < 0)
4417 : 0 : fdcache_mbs = 1024; // 1 gigabyte
4418 : : else
4419 : 72 : fdcache_mbs = sfs.f_bavail * sfs.f_bsize / 1024 / 1024 / 4; // 25% of free space
4420 : 72 : fdcache_mintmp = 25; // emergency flush at 25% remaining (75% full)
4421 : 72 : fdcache_prefetch = 64; // guesstimate storage is this much less costly than re-decompression
4422 : 72 : fdcache_fds = (concurrency + fdcache_prefetch) * 2;
4423 : :
4424 : : /* Parse and process arguments. */
4425 : 72 : int remaining;
4426 : 72 : (void) argp_parse (&argp, argc, argv, ARGP_IN_ORDER, &remaining, NULL);
4427 [ - + ]: 72 : if (remaining != argc)
4428 : 0 : error (EXIT_FAILURE, 0,
4429 : 0 : "unexpected argument: %s", argv[remaining]);
4430 : :
4431 : : // Make the prefetch cache spaces a fraction of the main fdcache if
4432 : : // unspecified.
4433 [ + + ]: 72 : if (fdcache_prefetch_fds == 0)
4434 : 70 : fdcache_prefetch_fds = fdcache_fds / 2;
4435 [ + + ]: 72 : if (fdcache_prefetch_mbs == 0)
4436 : 70 : fdcache_prefetch_mbs = fdcache_mbs / 2;
4437 : :
4438 [ + + + + : 72 : if (scan_archives.size()==0 && !scan_files && source_paths.size()>0)
- + ]
4439 [ # # ]: 0 : obatched(clog) << "warning: without -F -R -U -Z, ignoring PATHs" << endl;
4440 : :
4441 : 72 : fdcache.limit(fdcache_fds, fdcache_mbs, fdcache_prefetch_fds, fdcache_prefetch_mbs);
4442 : :
4443 : 72 : (void) signal (SIGPIPE, SIG_IGN); // microhttpd can generate it incidentally, ignore
4444 : 72 : (void) signal (SIGINT, signal_handler); // ^C
4445 : 72 : (void) signal (SIGHUP, signal_handler); // EOF
4446 : 72 : (void) signal (SIGTERM, signal_handler); // systemd
4447 : 72 : (void) signal (SIGUSR1, sigusr1_handler); // end-user
4448 : 72 : (void) signal (SIGUSR2, sigusr2_handler); // end-user
4449 : :
4450 : : /* Get database ready. */
4451 [ + + ]: 72 : if (! passive_p)
4452 : : {
4453 : 70 : rc = sqlite3_open_v2 (db_path.c_str(), &db, (SQLITE_OPEN_READWRITE
4454 : : |SQLITE_OPEN_URI
4455 : : |SQLITE_OPEN_PRIVATECACHE
4456 : : |SQLITE_OPEN_CREATE
4457 : : |SQLITE_OPEN_FULLMUTEX), /* thread-safe */
4458 : : NULL);
4459 [ - + ]: 70 : if (rc == SQLITE_CORRUPT)
4460 : : {
4461 : 0 : (void) unlink (db_path.c_str());
4462 : 0 : error (EXIT_FAILURE, 0,
4463 : : "cannot open %s, deleted database: %s", db_path.c_str(), sqlite3_errmsg(db));
4464 : : }
4465 [ - + ]: 70 : else if (rc)
4466 : : {
4467 : 0 : error (EXIT_FAILURE, 0,
4468 : : "cannot open %s, consider deleting database: %s", db_path.c_str(), sqlite3_errmsg(db));
4469 : : }
4470 : : }
4471 : :
4472 : : // open the readonly query variant
4473 : : // NB: PRIVATECACHE allows web queries to operate in parallel with
4474 : : // much other grooming/scanning operation.
4475 : 72 : rc = sqlite3_open_v2 (db_path.c_str(), &dbq, (SQLITE_OPEN_READONLY
4476 : : |SQLITE_OPEN_URI
4477 : : |SQLITE_OPEN_PRIVATECACHE
4478 : : |SQLITE_OPEN_FULLMUTEX), /* thread-safe */
4479 : : NULL);
4480 [ - + ]: 72 : if (rc)
4481 : : {
4482 : 0 : error (EXIT_FAILURE, 0,
4483 : : "cannot open %s, consider deleting database: %s", db_path.c_str(), sqlite3_errmsg(dbq));
4484 : : }
4485 : :
4486 : :
4487 [ + - ]: 144 : obatched(clog) << "opened database " << db_path
4488 [ + + + - : 74 : << (db?" rw":"") << (dbq?" ro":"") << endl;
- + + - +
- ]
4489 [ + - + - ]: 144 : obatched(clog) << "sqlite version " << sqlite3_version << endl;
4490 [ + + + - : 214 : obatched(clog) << "service mode " << (passive_p ? "passive":"active") << endl;
+ - ]
4491 : :
4492 : : // add special string-prefix-similarity function used in rpm sref/sdef resolution
4493 : 72 : rc = sqlite3_create_function(dbq, "sharedprefix", 2, SQLITE_UTF8, NULL,
4494 : : & sqlite3_sharedprefix_fn, NULL, NULL);
4495 [ - + ]: 72 : if (rc != SQLITE_OK)
4496 : 0 : error (EXIT_FAILURE, 0,
4497 : : "cannot create sharedprefix function: %s", sqlite3_errmsg(dbq));
4498 : :
4499 [ + + ]: 72 : if (! passive_p)
4500 : : {
4501 [ + + ]: 70 : if (verbose > 3)
4502 [ + - + - ]: 88 : obatched(clog) << "ddl: " << DEBUGINFOD_SQLITE_DDL << endl;
4503 : 70 : rc = sqlite3_exec (db, DEBUGINFOD_SQLITE_DDL, NULL, NULL, NULL);
4504 [ - + ]: 70 : if (rc != SQLITE_OK)
4505 : : {
4506 : 0 : error (EXIT_FAILURE, 0,
4507 : : "cannot run database schema ddl: %s", sqlite3_errmsg(db));
4508 : : }
4509 : : }
4510 : :
4511 [ + - + - : 144 : obatched(clog) << "libmicrohttpd version " << MHD_get_version() << endl;
+ - ]
4512 : :
4513 : : /* If '-C' wasn't given or was given with no arg, pick a reasonable default
4514 : : for the number of worker threads. */
4515 [ + + ]: 72 : if (connection_pool == 0)
4516 : 68 : connection_pool = default_concurrency();
4517 : :
4518 : : /* Note that MHD_USE_EPOLL and MHD_USE_THREAD_PER_CONNECTION don't
4519 : : work together. */
4520 : 72 : unsigned int use_epoll = 0;
4521 : : #if MHD_VERSION >= 0x00095100
4522 : 72 : use_epoll = MHD_USE_EPOLL;
4523 : : #endif
4524 : :
4525 : 72 : unsigned int mhd_flags = (
4526 : : #if MHD_VERSION >= 0x00095300
4527 : : MHD_USE_INTERNAL_POLLING_THREAD
4528 : : #else
4529 : : MHD_USE_SELECT_INTERNALLY
4530 : : #endif
4531 : : | MHD_USE_DUAL_STACK
4532 : : | use_epoll
4533 : : #if MHD_VERSION >= 0x00095200
4534 : : | MHD_USE_ITC
4535 : : #endif
4536 : : | MHD_USE_DEBUG); /* report errors to stderr */
4537 : :
4538 : : // Start httpd server threads. Use a single dual-homed pool.
4539 : 72 : MHD_Daemon *d46 = MHD_start_daemon (mhd_flags, http_port,
4540 : : NULL, NULL, /* default accept policy */
4541 : : handler_cb, NULL, /* handler callback */
4542 : : MHD_OPTION_EXTERNAL_LOGGER,
4543 : : error_cb, NULL,
4544 : : MHD_OPTION_THREAD_POOL_SIZE,
4545 : : (int)connection_pool,
4546 : : MHD_OPTION_END);
4547 : :
4548 : 72 : MHD_Daemon *d4 = NULL;
4549 [ - + ]: 72 : if (d46 == NULL)
4550 : : {
4551 : : // Cannot use dual_stack, use ipv4 only
4552 : 0 : mhd_flags &= ~(MHD_USE_DUAL_STACK);
4553 [ # # ]: 0 : d4 = MHD_start_daemon (mhd_flags, http_port,
4554 : : NULL, NULL, /* default accept policy */
4555 : : handler_cb, NULL, /* handler callback */
4556 : : MHD_OPTION_EXTERNAL_LOGGER,
4557 : : error_cb, NULL,
4558 : : (connection_pool
4559 : : ? MHD_OPTION_THREAD_POOL_SIZE
4560 : : : MHD_OPTION_END),
4561 : : (connection_pool
4562 : : ? (int)connection_pool
4563 : : : MHD_OPTION_END),
4564 : : MHD_OPTION_END);
4565 [ # # ]: 0 : if (d4 == NULL)
4566 : : {
4567 : 0 : sqlite3 *database = db;
4568 : 0 : sqlite3 *databaseq = dbq;
4569 : 0 : db = dbq = 0; // for signal_handler not to freak
4570 : 0 : sqlite3_close (databaseq);
4571 : 0 : sqlite3_close (database);
4572 : 0 : error (EXIT_FAILURE, 0, "cannot start http server at port %d",
4573 : : http_port);
4574 : : }
4575 : :
4576 : : }
4577 : 72 : obatched(clog) << "started http server on"
4578 : : << (d4 != NULL ? " IPv4 " : " IPv4 IPv6 ")
4579 [ + - + - : 144 : << "port=" << http_port << endl;
+ - + - +
- ]
4580 : :
4581 : : // add maxigroom sql if -G given
4582 [ - + ]: 72 : if (maxigroom)
4583 : : {
4584 [ # # ]: 0 : obatched(clog) << "maxigrooming database, please wait." << endl;
4585 [ # # ]: 0 : extra_ddl.push_back("create index if not exists " BUILDIDS "_r_sref_arc on " BUILDIDS "_r_sref(artifactsrc);");
4586 [ # # ]: 0 : extra_ddl.push_back("delete from " BUILDIDS "_r_sdef where not exists (select 1 from " BUILDIDS "_r_sref b where " BUILDIDS "_r_sdef.content = b.artifactsrc);");
4587 [ # # ]: 0 : extra_ddl.push_back("drop index if exists " BUILDIDS "_r_sref_arc;");
4588 : :
4589 : : // NB: we don't maxigroom the _files interning table. It'd require a temp index on all the
4590 : : // tables that have file foreign-keys, which is a lot.
4591 : :
4592 : : // NB: with =delete, may take up 3x disk space total during vacuum process
4593 : : // vs. =off (only 2x but may corrupt database if program dies mid-vacuum)
4594 : : // vs. =wal (>3x observed, but safe)
4595 [ # # ]: 0 : extra_ddl.push_back("pragma journal_mode=delete;");
4596 [ # # ]: 0 : extra_ddl.push_back("vacuum;");
4597 [ # # ]: 0 : extra_ddl.push_back("pragma journal_mode=wal;");
4598 : : }
4599 : :
4600 : : // run extra -D sql if given
4601 [ + + ]: 72 : if (! passive_p)
4602 [ - + ]: 70 : for (auto&& i: extra_ddl)
4603 : : {
4604 [ # # ]: 0 : if (verbose > 1)
4605 [ # # # # ]: 0 : obatched(clog) << "extra ddl:\n" << i << endl;
4606 : 0 : rc = sqlite3_exec (db, i.c_str(), NULL, NULL, NULL);
4607 [ # # # # ]: 0 : if (rc != SQLITE_OK && rc != SQLITE_DONE && rc != SQLITE_ROW)
4608 : 0 : error (0, 0,
4609 : : "warning: cannot run database extra ddl %s: %s", i.c_str(), sqlite3_errmsg(db));
4610 : :
4611 [ # # ]: 0 : if (maxigroom)
4612 [ # # ]: 0 : obatched(clog) << "maxigroomed database" << endl;
4613 : : }
4614 : :
4615 [ + + ]: 72 : if (! passive_p)
4616 [ + - + - ]: 140 : obatched(clog) << "search concurrency " << concurrency << endl;
4617 : 72 : obatched(clog) << "webapi connection pool " << connection_pool
4618 [ + - - + : 72 : << (connection_pool ? "" : " (unlimited)") << endl;
+ - + - ]
4619 [ + + ]: 72 : if (! passive_p) {
4620 [ + - + - ]: 140 : obatched(clog) << "rescan time " << rescan_s << endl;
4621 [ + - + - ]: 140 : obatched(clog) << "scan checkpoint " << scan_checkpoint << endl;
4622 : : }
4623 [ + - + - ]: 144 : obatched(clog) << "fdcache fds " << fdcache_fds << endl;
4624 [ + - + - ]: 144 : obatched(clog) << "fdcache mbs " << fdcache_mbs << endl;
4625 [ + - + - ]: 144 : obatched(clog) << "fdcache prefetch " << fdcache_prefetch << endl;
4626 [ + - + - ]: 144 : obatched(clog) << "fdcache tmpdir " << tmpdir << endl;
4627 [ + - + - ]: 144 : obatched(clog) << "fdcache tmpdir min% " << fdcache_mintmp << endl;
4628 [ + + ]: 72 : if (! passive_p)
4629 [ + - + - ]: 140 : obatched(clog) << "groom time " << groom_s << endl;
4630 [ + - + - ]: 144 : obatched(clog) << "prefetch fds " << fdcache_prefetch_fds << endl;
4631 [ + - + - ]: 144 : obatched(clog) << "prefetch mbs " << fdcache_prefetch_mbs << endl;
4632 [ + - + - ]: 144 : obatched(clog) << "forwarded ttl limit " << forwarded_ttl_limit << endl;
4633 : :
4634 [ + + ]: 72 : if (scan_archives.size()>0)
4635 : : {
4636 : 50 : obatched ob(clog);
4637 [ + - ]: 50 : auto& o = ob << "accepting archive types ";
4638 [ + + ]: 152 : for (auto&& arch : scan_archives)
4639 [ + - + - : 102 : o << arch.first << "(" << arch.second << ") ";
+ - + - ]
4640 [ + - ]: 50 : o << endl;
4641 : 50 : }
4642 : 72 : const char* du = getenv(DEBUGINFOD_URLS_ENV_VAR);
4643 [ + + + + ]: 72 : if (du && du[0] != '\0') // set to non-empty string?
4644 [ + - + - ]: 28 : obatched(clog) << "upstream debuginfod servers: " << du << endl;
4645 : :
4646 [ + + ]: 72 : vector<pthread_t> all_threads;
4647 : :
4648 [ + + ]: 72 : if (! passive_p)
4649 : : {
4650 : 70 : pthread_t pt;
4651 : 70 : rc = pthread_create (& pt, NULL, thread_main_groom, NULL);
4652 [ - + ]: 70 : if (rc)
4653 : 0 : error (EXIT_FAILURE, rc, "cannot spawn thread to groom database\n");
4654 : : else
4655 : : {
4656 : : #ifdef HAVE_PTHREAD_SETNAME_NP
4657 : 70 : (void) pthread_setname_np (pt, "groom");
4658 : : #endif
4659 [ + - ]: 70 : all_threads.push_back(pt);
4660 : : }
4661 : :
4662 [ + + + + ]: 70 : if (scan_files || scan_archives.size() > 0)
4663 : : {
4664 [ + - ]: 64 : if (scan_checkpoint > 0)
4665 [ + - ]: 64 : scan_barrier = new sqlite_checkpoint_pb(concurrency, (unsigned) scan_checkpoint);
4666 : :
4667 : 64 : rc = pthread_create (& pt, NULL, thread_main_fts_source_paths, NULL);
4668 [ - + ]: 64 : if (rc)
4669 : 0 : error (EXIT_FAILURE, rc, "cannot spawn thread to traverse source paths\n");
4670 : : #ifdef HAVE_PTHREAD_SETNAME_NP
4671 : 64 : (void) pthread_setname_np (pt, "traverse");
4672 : : #endif
4673 [ + - ]: 64 : all_threads.push_back(pt);
4674 : :
4675 [ + + ]: 320 : for (unsigned i=0; i<concurrency; i++)
4676 : : {
4677 : 256 : rc = pthread_create (& pt, NULL, thread_main_scanner, NULL);
4678 [ - + ]: 256 : if (rc)
4679 : 0 : error (EXIT_FAILURE, rc, "cannot spawn thread to scan source files / archives\n");
4680 : : #ifdef HAVE_PTHREAD_SETNAME_NP
4681 : 256 : (void) pthread_setname_np (pt, "scan");
4682 : : #endif
4683 [ + - ]: 256 : all_threads.push_back(pt);
4684 : : }
4685 : : }
4686 : : }
4687 : :
4688 : : /* Trivial main loop! */
4689 [ + - + - ]: 72 : set_metric("ready", 1);
4690 [ + + ]: 212 : while (! interrupted)
4691 [ + - ]: 140 : pause ();
4692 [ + - ]: 72 : scanq.nuke(); // wake up any remaining scanq-related threads, let them die
4693 [ + + + - ]: 72 : if (scan_barrier) scan_barrier->nuke(); // ... in case they're stuck in a barrier
4694 [ + - + - ]: 72 : set_metric("ready", 0);
4695 : :
4696 [ + - ]: 72 : if (verbose)
4697 [ + - + - : 144 : obatched(clog) << "stopping" << endl;
- - ]
4698 : :
4699 : : /* Join all our threads. */
4700 [ + + ]: 462 : for (auto&& it : all_threads)
4701 [ + - ]: 390 : pthread_join (it, NULL);
4702 : :
4703 : : /* Stop all the web service threads. */
4704 [ + - + - ]: 72 : if (d46) MHD_stop_daemon (d46);
4705 [ - + - - ]: 72 : if (d4) MHD_stop_daemon (d4);
4706 : :
4707 [ + + ]: 72 : if (! passive_p)
4708 : : {
4709 : : /* With all threads known dead, we can clean up the global resources. */
4710 [ + - ]: 70 : rc = sqlite3_exec (db, DEBUGINFOD_SQLITE_CLEANUP_DDL, NULL, NULL, NULL);
4711 [ - + ]: 70 : if (rc != SQLITE_OK)
4712 : : {
4713 [ # # # # ]: 0 : error (0, 0,
4714 : : "warning: cannot run database cleanup ddl: %s", sqlite3_errmsg(db));
4715 : : }
4716 : : }
4717 : :
4718 [ + - ]: 72 : debuginfod_pool_groom ();
4719 [ + + ]: 72 : delete scan_barrier;
4720 : :
4721 : : // NB: no problem with unconditional free here - an earlier failed regcomp would exit program
4722 [ + - ]: 72 : (void) regfree (& file_include_regex);
4723 [ + - ]: 72 : (void) regfree (& file_exclude_regex);
4724 : :
4725 : 72 : sqlite3 *database = db;
4726 : 72 : sqlite3 *databaseq = dbq;
4727 : 72 : db = dbq = 0; // for signal_handler not to freak
4728 [ + - ]: 72 : (void) sqlite3_close (databaseq);
4729 [ + + ]: 72 : if (! passive_p)
4730 [ + - ]: 70 : (void) sqlite3_close (database);
4731 : :
4732 [ + + ]: 72 : return 0;
4733 : 72 : }
|
