1 /*
2 * Copyright (c) 2013-2022 Douglas Gilbert.
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 *
26 * SPDX-License-Identifier: BSD-2-Clause
27 */
28
29 #include <iostream>
30 #include <vector>
31 #include <system_error>
32 #include <thread>
33 #include <mutex>
34 #include <chrono>
35
36 #include <unistd.h>
37 #include <fcntl.h>
38 #include <stdio.h>
39 #include <stdlib.h>
40 #include <string.h>
41 #include <errno.h>
42 #include <ctype.h>
43 #include <sys/ioctl.h>
44 #include <sys/types.h>
45 #include <sys/stat.h>
46
47 #include "sg_lib.h"
48 #include "sg_pt.h"
49 #include "sg_unaligned.h"
50
51 static const char * version_str = "1.11 20220425";
52 static const char * util_name = "sg_tst_excl3";
53
54 /* This is a test program for checking O_EXCL on open() works. It uses
55 * multiple threads and can be run as multiple processes and attempts
56 * to "break" O_EXCL. The strategy is to open a device O_EXCL|O_NONBLOCK
57 * and do a double increment on a LB then close it from a single thread.
58 * the remaining threads open that device O_NONBLOCK and do a read and
59 * note if the number is odd. Assuming the count starts as an even
60 * (typically 0) then it should remain even. Odd instances
61 * are counted and reported at the end of the program, after all threads
62 * have completed.
63 *
64 * This is C++ code with some things from C++11 (e.g. threads) and was
65 * only just able to compile (when some things were reverted) with gcc/g++
66 * version 4.7.3 found in Ubuntu 13.04 . C++11 "feature complete" support
67 * was not available until g++ version 4.8.1 and that is found in Fedora
68 * 19 and Ubuntu 13.10 .
69 *
70 * The build uses various object files from the <sg3_utils>/lib directory
71 * which is assumed to be a sibling of this examples directory. Those
72 * object files in the lib directory can be built with:
73 * cd <sg3_utils> ; ./configure ; cd lib; make
74 * Then:
75 * cd ../testing
76 * make sg_tst_excl3
77 *
78 * BEWARE: this utility modifies a logical block (default LBA 1000) on the
79 * given device.
80 *
81 */
82
83 using namespace std;
84 using namespace std::chrono;
85
86 #define DEF_NUM_PER_THREAD 200
87 #define DEF_NUM_THREADS 4
88 #define DEF_WAIT_MS 0 /* 0: yield; -1: don't wait; -2: sleep(0) */
89
90 #define DEF_LBA 1000
91
92 #define EBUFF_SZ 256
93
94
95 static mutex odd_count_mutex;
96 static mutex console_mutex;
97 static unsigned int odd_count;
98 static unsigned int ebusy_count;
99
100
101 static void
usage(void)102 usage(void)
103 {
104 printf("Usage: %s [-b] [-f] [-h] [-l <lba>] [-n <n_per_thr>]\n"
105 " [-R] [-t <num_thrs>] [-V] [-w <wait_ms>] "
106 "[-x]\n"
107 " <disk_device>\n", util_name);
108 printf(" where\n");
109 printf(" -b block on open (def: O_NONBLOCK)\n");
110 printf(" -f force: any SCSI disk (def: only "
111 "scsi_debug)\n");
112 printf(" WARNING: <lba> written to\n");
113 printf(" -h print this usage message then exit\n");
114 printf(" -l <lba> logical block to increment (def: %u)\n",
115 DEF_LBA);
116 printf(" -n <n_per_thr> number of loops per thread "
117 "(def: %d)\n", DEF_NUM_PER_THREAD);
118 printf(" -R all readers; so first thread (id=0) "
119 "just reads\n");
120 printf(" -t <num_thrs> number of threads (def: %d)\n",
121 DEF_NUM_THREADS);
122 printf(" -V print version number then exit\n");
123 printf(" -w <wait_ms> >0: sleep_for(<wait_ms>); =0: "
124 "yield(); -1: no\n"
125 " wait; -2: sleep(0) (def: %d)\n",
126 DEF_WAIT_MS);
127 printf(" -x don't use O_EXCL on first thread "
128 "(def: use\n"
129 " O_EXCL on first thread)\n\n");
130 printf("Test O_EXCL open flag with pass-through drivers. First thread "
131 "(id=0) does\nopen/close cycle with the O_EXCL flag then does a "
132 "double increment on\nlba (using its first 4 bytes). Remaining "
133 "theads read (without\nO_EXCL flag on open) and check the "
134 "value is even.\n");
135 }
136
137 /* Assumed a lock (mutex) held when pt_err() is called */
138 static int
pt_err(int res)139 pt_err(int res)
140 {
141 if (res < 0)
142 fprintf(stderr, " pass through os error: %s\n", safe_strerror(-res));
143 else if (SCSI_PT_DO_BAD_PARAMS == res)
144 fprintf(stderr, " bad pass through setup\n");
145 else if (SCSI_PT_DO_TIMEOUT == res)
146 fprintf(stderr, " pass through timeout\n");
147 else
148 fprintf(stderr, " do_scsi_pt error=%d\n", res);
149 return -1;
150 }
151
152 /* Assumed a lock (mutex) held when pt_cat_no_good() is called */
153 static int
pt_cat_no_good(int cat,struct sg_pt_base * ptp,const unsigned char * sbp)154 pt_cat_no_good(int cat, struct sg_pt_base * ptp, const unsigned char * sbp)
155 {
156 int slen;
157 char b[256];
158 const int bl = (int)sizeof(b);
159
160 switch (cat) {
161 case SCSI_PT_RESULT_STATUS: /* other than GOOD and CHECK CONDITION */
162 sg_get_scsi_status_str(get_scsi_pt_status_response(ptp), bl, b);
163 fprintf(stderr, " scsi status: %s\n", b);
164 break;
165 case SCSI_PT_RESULT_SENSE:
166 slen = get_scsi_pt_sense_len(ptp);
167 sg_get_sense_str("", sbp, slen, 1, bl, b);
168 fprintf(stderr, "%s", b);
169 break;
170 case SCSI_PT_RESULT_TRANSPORT_ERR:
171 get_scsi_pt_transport_err_str(ptp, bl, b);
172 fprintf(stderr, " transport: %s", b);
173 break;
174 case SCSI_PT_RESULT_OS_ERR:
175 get_scsi_pt_os_err_str(ptp, bl, b);
176 fprintf(stderr, " os: %s", b);
177 break;
178 default:
179 fprintf(stderr, " unknown pt result category (%d)\n", cat);
180 break;
181 }
182 return -1;
183 }
184
185 #define READ16_REPLY_LEN 512
186 #define READ16_CMD_LEN 16
187 #define WRITE16_REPLY_LEN 512
188 #define WRITE16_CMD_LEN 16
189
190 /* Opens dev_name and spins if busy (i.e. gets EBUSY), sleeping for
191 * wait_ms milliseconds if wait_ms is positive. Reads lba and treats the
192 * first 4 bytes as an int (SCSI endian), increments it and writes it back.
193 * Repeats so that happens twice. Then closes dev_name. If an error occurs
194 * returns -1 else returns 0 if first int read is even otherwise returns 1. */
195 static int
do_rd_inc_wr_twice(const char * dev_name,int read_only,unsigned int lba,int block,int excl,int wait_ms,unsigned int & ebusys)196 do_rd_inc_wr_twice(const char * dev_name, int read_only, unsigned int lba,
197 int block, int excl, int wait_ms, unsigned int & ebusys)
198 {
199 int k, sg_fd, res, cat;
200 int odd = 0;
201 unsigned int u = 0;
202 struct sg_pt_base * ptp = NULL;
203 unsigned char r16CmdBlk [READ16_CMD_LEN] =
204 {0x88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0};
205 unsigned char w16CmdBlk [WRITE16_CMD_LEN] =
206 {0x8a, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0};
207 unsigned char sense_buffer[64] SG_C_CPP_ZERO_INIT;
208 unsigned char lb[READ16_REPLY_LEN];
209 char ebuff[EBUFF_SZ];
210 int open_flags = O_RDWR;
211
212 sg_put_unaligned_be64(lba, r16CmdBlk + 2);
213 sg_put_unaligned_be64(lba, w16CmdBlk + 2);
214 if (! block)
215 open_flags |= O_NONBLOCK;
216 if (excl)
217 open_flags |= O_EXCL;
218
219 while (((sg_fd = scsi_pt_open_flags(dev_name, open_flags, 0)) < 0) &&
220 (-EBUSY == sg_fd)) {
221 ++ebusys;
222 if (wait_ms > 0)
223 this_thread::sleep_for(milliseconds{wait_ms});
224 else if (0 == wait_ms)
225 this_thread::yield(); // thread yield
226 else if (-2 == wait_ms)
227 sleep(0); // process yield ??
228 }
229 if (sg_fd < 0) {
230 snprintf(ebuff, EBUFF_SZ,
231 "do_rd_inc_wr_twice: error opening file: %s", dev_name);
232 {
233 lock_guard<mutex> lg(console_mutex);
234
235 perror(ebuff);
236 }
237 return -1;
238 }
239
240 ptp = construct_scsi_pt_obj();
241 for (k = 0; k < 2; ++k) {
242 /* Prepare READ_16 command */
243 clear_scsi_pt_obj(ptp);
244 set_scsi_pt_cdb(ptp, r16CmdBlk, sizeof(r16CmdBlk));
245 set_scsi_pt_sense(ptp, sense_buffer, sizeof(sense_buffer));
246 set_scsi_pt_data_in(ptp, lb, READ16_REPLY_LEN);
247 res = do_scsi_pt(ptp, sg_fd, 20 /* secs timeout */, 1);
248 if (res) {
249 {
250 lock_guard<mutex> lg(console_mutex);
251
252 fprintf(stderr, "READ_16 do_scsi_pt() submission error\n");
253 res = pt_err(res);
254 }
255 goto err;
256 }
257 cat = get_scsi_pt_result_category(ptp);
258 if (SCSI_PT_RESULT_GOOD != cat) {
259 {
260 lock_guard<mutex> lg(console_mutex);
261
262 fprintf(stderr, "READ_16 do_scsi_pt() category problem\n");
263 res = pt_cat_no_good(cat, ptp, sense_buffer);
264 }
265 goto err;
266 }
267
268 u = sg_get_unaligned_be32(lb);
269 // Assuming u starts test as even (probably 0), expect it to stay even
270 if (0 == k)
271 odd = (1 == (u % 2));
272
273 if (wait_ms > 0) /* allow daylight for bad things ... */
274 this_thread::sleep_for(milliseconds{wait_ms});
275 else if (0 == wait_ms)
276 this_thread::yield(); // thread yield
277 else if (-2 == wait_ms)
278 sleep(0); // process yield ??
279
280 if (read_only)
281 break;
282 ++u;
283 sg_put_unaligned_be32(u, lb);
284
285 /* Prepare WRITE_16 command */
286 clear_scsi_pt_obj(ptp);
287 set_scsi_pt_cdb(ptp, w16CmdBlk, sizeof(w16CmdBlk));
288 set_scsi_pt_sense(ptp, sense_buffer, sizeof(sense_buffer));
289 set_scsi_pt_data_out(ptp, lb, WRITE16_REPLY_LEN);
290 res = do_scsi_pt(ptp, sg_fd, 20 /* secs timeout */, 1);
291 if (res) {
292 {
293 lock_guard<mutex> lg(console_mutex);
294
295 fprintf(stderr, "WRITE_16 do_scsi_pt() submission error\n");
296 res = pt_err(res);
297 }
298 goto err;
299 }
300 cat = get_scsi_pt_result_category(ptp);
301 if (SCSI_PT_RESULT_GOOD != cat) {
302 {
303 lock_guard<mutex> lg(console_mutex);
304
305 fprintf(stderr, "WRITE_16 do_scsi_pt() category problem\n");
306 res = pt_cat_no_good(cat, ptp, sense_buffer);
307 }
308 goto err;
309 }
310 }
311 err:
312 if (ptp)
313 destruct_scsi_pt_obj(ptp);
314 scsi_pt_close_device(sg_fd);
315 return odd;
316 }
317
318
319 #define INQ_REPLY_LEN 96
320 #define INQ_CMD_LEN 6
321
322 /* Send INQUIRY and fetches response. If okay puts PRODUCT ID field
323 * in b (up to m_blen bytes). Does not use O_EXCL flag. Returns 0 on success,
324 * else -1 . */
325 static int
do_inquiry_prod_id(const char * dev_name,int block,int wait_ms,unsigned int & ebusys,char * b,int b_mlen)326 do_inquiry_prod_id(const char * dev_name, int block, int wait_ms,
327 unsigned int & ebusys, char * b, int b_mlen)
328 {
329 int sg_fd, res, cat;
330 struct sg_pt_base * ptp = NULL;
331 unsigned char inqCmdBlk [INQ_CMD_LEN] =
332 {0x12, 0, 0, 0, INQ_REPLY_LEN, 0};
333 unsigned char inqBuff[INQ_REPLY_LEN];
334 unsigned char sense_buffer[64] SG_C_CPP_ZERO_INIT;
335 char ebuff[EBUFF_SZ];
336 int open_flags = O_RDWR; /* since O_EXCL | O_RDONLY gives EPERM */
337
338 if (! block)
339 open_flags |= O_NONBLOCK;
340 while (((sg_fd = scsi_pt_open_flags(dev_name, open_flags, 0)) < 0) &&
341 (-EBUSY == sg_fd)) {
342 ++ebusys;
343 if (wait_ms > 0)
344 this_thread::sleep_for(milliseconds{wait_ms});
345 else if (0 == wait_ms)
346 this_thread::yield(); // thread yield
347 else if (-2 == wait_ms)
348 sleep(0); // process yield ??
349 }
350 if (sg_fd < 0) {
351 snprintf(ebuff, EBUFF_SZ,
352 "do_inquiry_prod_id: error opening file: %s", dev_name);
353 perror(ebuff);
354 return -1;
355 }
356 /* Prepare INQUIRY command */
357 ptp = construct_scsi_pt_obj();
358 clear_scsi_pt_obj(ptp);
359 set_scsi_pt_cdb(ptp, inqCmdBlk, sizeof(inqCmdBlk));
360 set_scsi_pt_sense(ptp, sense_buffer, sizeof(sense_buffer));
361 set_scsi_pt_data_in(ptp, inqBuff, INQ_REPLY_LEN);
362 res = do_scsi_pt(ptp, sg_fd, 20 /* secs timeout */, 1);
363 if (res) {
364 fprintf(stderr, "INQUIRY do_scsi_pt() submission error\n");
365 res = pt_err(res);
366 goto err;
367 }
368 cat = get_scsi_pt_result_category(ptp);
369 if (SCSI_PT_RESULT_GOOD != cat) {
370 fprintf(stderr, "INQUIRY do_scsi_pt() category problem\n");
371 res = pt_cat_no_good(cat, ptp, sense_buffer);
372 goto err;
373 }
374
375 /* Good, so fetch Product ID from response, copy to 'b' */
376 if (b_mlen > 0) {
377 if (b_mlen > 16) {
378 memcpy(b, inqBuff + 16, 16);
379 b[16] = '\0';
380 } else {
381 memcpy(b, inqBuff + 16, b_mlen - 1);
382 b[b_mlen - 1] = '\0';
383 }
384 }
385 err:
386 if (ptp)
387 destruct_scsi_pt_obj(ptp);
388 close(sg_fd);
389 return res;
390 }
391
392 static void
work_thread(const char * dev_name,unsigned int lba,int id,int block,int excl,bool all_readers,int num,int wait_ms)393 work_thread(const char * dev_name, unsigned int lba, int id, int block,
394 int excl, bool all_readers, int num, int wait_ms)
395 {
396 unsigned int thr_odd_count = 0;
397 unsigned int thr_ebusy_count = 0;
398 int k, res;
399 int reader = ((id > 0) || (all_readers));
400
401 {
402 lock_guard<mutex> lg(console_mutex);
403
404 cerr << "Enter work_thread id=" << id << " excl=" << excl << " block="
405 << block << " reader=" << reader << endl;
406 }
407 for (k = 0; k < num; ++k) {
408 res = do_rd_inc_wr_twice(dev_name, reader, lba, block, excl,
409 wait_ms, thr_ebusy_count);
410 if (res < 0)
411 break;
412 if (res)
413 ++thr_odd_count;
414 }
415 {
416 lock_guard<mutex> lg(console_mutex);
417
418 if (k < num)
419 cerr << "thread id=" << id << " FAILed at iteration: " << k
420 << '\n';
421 else
422 cerr << "thread id=" << id << " normal exit" << '\n';
423 }
424
425 {
426 lock_guard<mutex> lg(odd_count_mutex);
427
428 odd_count += thr_odd_count;
429 ebusy_count += thr_ebusy_count;
430 }
431 }
432
433
434 int
main(int argc,char * argv[])435 main(int argc, char * argv[])
436 {
437 int k, res;
438 int block = 0;
439 int force = 0;
440 unsigned int lba = DEF_LBA;
441 int num_per_thread = DEF_NUM_PER_THREAD;
442 bool all_readers = false;
443 int num_threads = DEF_NUM_THREADS;
444 int wait_ms = DEF_WAIT_MS;
445 int exclude_o_excl = 0;
446 char * dev_name = NULL;
447 char b[64];
448
449 for (k = 1; k < argc; ++k) {
450 if (0 == memcmp("-b", argv[k], 2))
451 ++block;
452 else if (0 == memcmp("-f", argv[k], 2))
453 ++force;
454 else if (0 == memcmp("-h", argv[k], 2)) {
455 usage();
456 return 0;
457 } else if (0 == memcmp("-l", argv[k], 2)) {
458 ++k;
459 if ((k < argc) && isdigit(*argv[k]))
460 lba = (unsigned int)atoi(argv[k]);
461 else
462 break;
463 } else if (0 == memcmp("-n", argv[k], 2)) {
464 ++k;
465 if ((k < argc) && isdigit(*argv[k]))
466 num_per_thread = atoi(argv[k]);
467 else
468 break;
469 } else if (0 == memcmp("-t", argv[k], 2)) {
470 ++k;
471 if ((k < argc) && isdigit(*argv[k]))
472 num_threads = atoi(argv[k]);
473 else
474 break;
475 } else if (0 == memcmp("-R", argv[k], 2))
476 all_readers = true;
477 else if (0 == memcmp("-V", argv[k], 2)) {
478 printf("%s version: %s\n", util_name, version_str);
479 return 0;
480 } else if (0 == memcmp("-w", argv[k], 2)) {
481 ++k;
482 if ((k < argc) && (isdigit(*argv[k]) || ('-' == *argv[k]))) {
483 if ('-' == *argv[k])
484 wait_ms = - atoi(argv[k] + 1);
485 else
486 wait_ms = atoi(argv[k]);
487 } else
488 break;
489 } else if (0 == memcmp("-x", argv[k], 2))
490 ++exclude_o_excl;
491 else if (*argv[k] == '-') {
492 printf("Unrecognized switch: %s\n", argv[k]);
493 dev_name = NULL;
494 break;
495 }
496 else if (! dev_name)
497 dev_name = argv[k];
498 else {
499 printf("too many arguments\n");
500 dev_name = 0;
501 break;
502 }
503 }
504 if (0 == dev_name) {
505 usage();
506 return 1;
507 }
508 try {
509
510 if (! force) {
511 res = do_inquiry_prod_id(dev_name, block, wait_ms, ebusy_count,
512 b, sizeof(b));
513 if (res) {
514 fprintf(stderr, "INQUIRY failed on %s\n", dev_name);
515 return 1;
516 }
517 // For safety, since <lba> written to, only permit scsi_debug
518 // devices. Bypass this with '-f' option.
519 if (0 != memcmp("scsi_debug", b, 10)) {
520 fprintf(stderr, "Since this utility writes to LBA %d, only "
521 "devices with scsi_debug\nproduct ID accepted.\n",
522 lba);
523 return 2;
524 }
525 }
526
527 vector<thread *> vt;
528
529 for (k = 0; k < num_threads; ++k) {
530 int excl = ((0 == k) && (! exclude_o_excl)) ? 1 : 0;
531
532 thread * tp = new thread {work_thread, dev_name, lba, k, block,
533 excl, all_readers, num_per_thread,
534 wait_ms};
535 vt.push_back(tp);
536 }
537
538 for (k = 0; k < (int)vt.size(); ++k)
539 vt[k]->join();
540
541 for (k = 0; k < (int)vt.size(); ++k)
542 delete vt[k];
543
544 cout << "Expecting odd count of 0, got " << odd_count << endl;
545 cout << "Number of EBUSYs: " << ebusy_count << endl;
546
547 }
548 catch(system_error& e) {
549 cerr << "got a system_error exception: " << e.what() << '\n';
550 auto ec = e.code();
551 cerr << "category: " << ec.category().name() << '\n';
552 cerr << "value: " << ec.value() << '\n';
553 cerr << "message: " << ec.message() << '\n';
554 cerr << "\nNote: if g++ may need '-pthread' or similar in "
555 "compile/link line" << '\n';
556 }
557 catch(...) {
558 cerr << "got another exception: " << '\n';
559 }
560 return 0;
561 }
562