From c31a245665c7bb7167edb50083e76c1d60130947 Mon Sep 17 00:00:00 2001 From: AlexeyAB Date: Wed, 9 Jan 2019 00:11:59 +0300 Subject: [PATCH] Experimental. Use -json_port 8070 to allow connection to the port:8070 to get json-stream --- .../darknet/x64/darknet_demo_json_stream.cmd | 7 + .../darknet/x64/darknet_demo_mjpeg_stream.cmd | 2 +- include/darknet.h | 3 +- src/coco.c | 5 +- src/demo.c | 13 +- src/demo.h | 3 +- src/detector.c | 3 +- src/http_stream.cpp | 161 +++++++++++++++++- src/http_stream.h | 1 + src/network.c | 38 +++++ src/yolo.c | 3 +- 11 files changed, 229 insertions(+), 10 deletions(-) create mode 100644 build/darknet/x64/darknet_demo_json_stream.cmd diff --git a/build/darknet/x64/darknet_demo_json_stream.cmd b/build/darknet/x64/darknet_demo_json_stream.cmd new file mode 100644 index 00000000..22a8859b --- /dev/null +++ b/build/darknet/x64/darknet_demo_json_stream.cmd @@ -0,0 +1,7 @@ +rem Run this file and then open URL in Chrome/Firefox: rem http://localhost:8070 +rem Or open: http://ip-address:8070 + +darknet.exe detector demo cfg/coco.data cfg/yolov3.cfg yolov3.weights test.mp4 -i 0 -json_port 8070 -dont_show -ext_output + + +pause \ No newline at end of file diff --git a/build/darknet/x64/darknet_demo_mjpeg_stream.cmd b/build/darknet/x64/darknet_demo_mjpeg_stream.cmd index a2c5f891..95bfb3b4 100644 --- a/build/darknet/x64/darknet_demo_mjpeg_stream.cmd +++ b/build/darknet/x64/darknet_demo_mjpeg_stream.cmd @@ -1,7 +1,7 @@ rem Run this file and then open URL in Chrome/Firefox: rem http://localhost:8090 rem Or open: http://ip-address:8090 -darknet.exe detector demo data/voc.data cfg/yolov2-voc.cfg yolo-voc.weights test.mp4 -i 0 -http_port 8090 -dont_show +darknet.exe detector demo cfg/coco.data cfg/yolov3.cfg yolov3.weights test.mp4 -i 0 -http_port 8090 -dont_show -ext_output pause \ No newline at end of file diff --git a/include/darknet.h b/include/darknet.h index 80dc28cd..b7e45f16 100644 --- a/include/darknet.h +++ b/include/darknet.h @@ -744,9 +744,10 @@ LIB_API detection *get_network_boxes(network *net, int w, int h, float thresh, f LIB_API void free_detections(detection *dets, int n); LIB_API void fuse_conv_batchnorm(network net); LIB_API void calculate_binary_weights(network net); +LIB_API char *detection_to_json(detection *dets, int nboxes, int classes, char **names, long long int frame_id); LIB_API layer* get_network_layer(network* net, int i); -LIB_API detection *get_network_boxes(network *net, int w, int h, float thresh, float hier, int *map, int relative, int *num, int letter); +//LIB_API detection *get_network_boxes(network *net, int w, int h, float thresh, float hier, int *map, int relative, int *num, int letter); LIB_API detection *make_network_boxes(network *net, float thresh, int *num); LIB_API void reset_rnn(network *net); LIB_API float *network_predict_image(network *net, image im); diff --git a/src/coco.c b/src/coco.c index 1913a478..62f6770a 100644 --- a/src/coco.c +++ b/src/coco.c @@ -222,7 +222,7 @@ void validate_coco(char *cfgfile, char *weightfile) free_image(val_resized[t]); } } - fseek(fp, -2, SEEK_CUR); + fseek(fp, -2, SEEK_CUR); fprintf(fp, "\n]\n"); fclose(fp); @@ -366,6 +366,7 @@ void run_coco(int argc, char **argv) { int dont_show = find_arg(argc, argv, "-dont_show"); int http_stream_port = find_int_arg(argc, argv, "-http_port", -1); + int json_port = find_int_arg(argc, argv, "-json_port", -1); char *out_filename = find_char_arg(argc, argv, "-out_filename", 0); char *prefix = find_char_arg(argc, argv, "-prefix", 0); float thresh = find_float_arg(argc, argv, "-thresh", .2); @@ -387,5 +388,5 @@ void run_coco(int argc, char **argv) else if(0==strcmp(argv[2], "valid")) validate_coco(cfg, weights); else if(0==strcmp(argv[2], "recall")) validate_coco_recall(cfg, weights); else if(0==strcmp(argv[2], "demo")) demo(cfg, weights, thresh, hier_thresh, cam_index, filename, coco_classes, 80, frame_skip, - prefix, out_filename, http_stream_port, dont_show, ext_output); + prefix, out_filename, http_stream_port, json_port, dont_show, ext_output); } diff --git a/src/demo.c b/src/demo.c index d57fae40..3aca9705 100644 --- a/src/demo.c +++ b/src/demo.c @@ -41,6 +41,8 @@ static int cpp_video_capture = 0; static float fps = 0; static float demo_thresh = 0; static int demo_ext_output = 0; +static long long int frame_id = 0; +static int demo_json_port = -1; static float *predictions[FRAMES]; static int demo_index = 0; @@ -114,6 +116,12 @@ void *detect_in_thread(void *ptr) det_img = ipl_images[(demo_index + FRAMES / 2 + 1) % FRAMES]; demo_index = (demo_index + 1)%FRAMES; + ++frame_id; + if (demo_json_port > 0) { + int timeout = 200; + send_json(dets, nboxes, l.classes, demo_names, frame_id, demo_json_port, timeout); + } + draw_detections_cv_v3(det_img, dets, nboxes, demo_thresh, demo_names, demo_alphabet, demo_classes, demo_ext_output); free_detections(dets, nboxes); @@ -130,7 +138,7 @@ double get_wall_time() } void demo(char *cfgfile, char *weightfile, float thresh, float hier_thresh, int cam_index, const char *filename, char **names, int classes, - int frame_skip, char *prefix, char *out_filename, int http_stream_port, int dont_show, int ext_output) + int frame_skip, char *prefix, char *out_filename, int http_stream_port, int json_port, int dont_show, int ext_output) { //skip = frame_skip; image **alphabet = load_alphabet(); @@ -140,6 +148,7 @@ void demo(char *cfgfile, char *weightfile, float thresh, float hier_thresh, int demo_classes = classes; demo_thresh = thresh; demo_ext_output = ext_output; + demo_json_port = json_port; printf("Demo\n"); net = parse_network_cfg_custom(cfgfile, 1); // set batch=1 if(weightfile){ @@ -355,7 +364,7 @@ void demo(char *cfgfile, char *weightfile, float thresh, float hier_thresh, int } #else void demo(char *cfgfile, char *weightfile, float thresh, float hier_thresh, int cam_index, const char *filename, char **names, int classes, - int frame_skip, char *prefix, char *out_filename, int http_stream_port, int dont_show, int ext_output) + int frame_skip, char *prefix, char *out_filename, int http_stream_port, int json_port, int dont_show, int ext_output) { fprintf(stderr, "Demo needs OpenCV for webcam images.\n"); } diff --git a/src/demo.h b/src/demo.h index 7ad7addf..c6b5246e 100644 --- a/src/demo.h +++ b/src/demo.h @@ -2,6 +2,7 @@ #define DEMO #include "image.h" -void demo(char *cfgfile, char *weightfile, float thresh, float hier_thresh, int cam_index, const char *filename, char **names, int classes, int frame_skip, char *prefix, char *out_filename, int http_stream_port, int dont_show, int ext_output); +void demo(char *cfgfile, char *weightfile, float thresh, float hier_thresh, int cam_index, const char *filename, char **names, int classes, + int frame_skip, char *prefix, char *out_filename, int http_stream_port, int json_port, int dont_show, int ext_output); #endif diff --git a/src/detector.c b/src/detector.c index f3b60f3f..3ae5c0fa 100644 --- a/src/detector.c +++ b/src/detector.c @@ -1352,6 +1352,7 @@ void run_detector(int argc, char **argv) int calc_map = find_arg(argc, argv, "-map"); check_mistakes = find_arg(argc, argv, "-check_mistakes"); int http_stream_port = find_int_arg(argc, argv, "-http_port", -1); + int json_port = find_int_arg(argc, argv, "-json_port", -1); char *out_filename = find_char_arg(argc, argv, "-out_filename", 0); char *outfile = find_char_arg(argc, argv, "-out", 0); char *prefix = find_char_arg(argc, argv, "-prefix", 0); @@ -1419,7 +1420,7 @@ void run_detector(int argc, char **argv) if (strlen(filename) > 0) if (filename[strlen(filename) - 1] == 0x0d) filename[strlen(filename) - 1] = 0; demo(cfg, weights, thresh, hier_thresh, cam_index, filename, names, classes, frame_skip, prefix, out_filename, - http_stream_port, dont_show, ext_output); + http_stream_port, json_port, dont_show, ext_output); free_list_contents_kvp(options); free_list(options); diff --git a/src/http_stream.cpp b/src/http_stream.cpp index c6ace438..71c6e2be 100644 --- a/src/http_stream.cpp +++ b/src/http_stream.cpp @@ -196,7 +196,8 @@ public: }; // ---------------------------------------- -void send_mjpeg(IplImage* ipl, int port, int timeout, int quality) { +void send_mjpeg(IplImage* ipl, int port, int timeout, int quality) +{ static MJPGWriter wri(port, timeout, quality); cv::Mat mat = cv::cvarrToMat(ipl); wri.write(mat); @@ -204,6 +205,164 @@ void send_mjpeg(IplImage* ipl, int port, int timeout, int quality) { } // ---------------------------------------- +class JSON_sender +{ + SOCKET sock; + SOCKET maxfd; + fd_set master; + int timeout; // master sock timeout, shutdown after timeout millis. + + int _write(int sock, char const*const s, int len) + { + if (len < 1) { len = strlen(s); } + return ::send(sock, s, len, 0); + } + +public: + + JSON_sender(int port = 0, int _timeout = 200000) + : sock(INVALID_SOCKET) + , timeout(_timeout) + { + FD_ZERO(&master); + if (port) + open(port); + } + + ~JSON_sender() + { + release(); + } + + bool release() + { + if (sock != INVALID_SOCKET) + ::shutdown(sock, 2); + sock = (INVALID_SOCKET); + return false; + } + + bool open(int port) + { + sock = ::socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); + + SOCKADDR_IN address; + address.sin_addr.s_addr = INADDR_ANY; + address.sin_family = AF_INET; + address.sin_port = htons(port); // ::htons(port); + if (::bind(sock, (SOCKADDR*)&address, sizeof(SOCKADDR_IN)) == SOCKET_ERROR) + { + cerr << "error : couldn't bind sock " << sock << " to port " << port << "!" << endl; + return release(); + } + if (::listen(sock, 10) == SOCKET_ERROR) + { + cerr << "error : couldn't listen on sock " << sock << " on port " << port << " !" << endl; + return release(); + } + FD_ZERO(&master); + FD_SET(sock, &master); + maxfd = sock; + return true; + } + + bool isOpened() + { + return sock != INVALID_SOCKET; + } + + bool write(char *outputbuf) + { + fd_set rread = master; + struct timeval to = { 0,timeout }; + if (::select(maxfd + 1, &rread, NULL, NULL, &to) <= 0) + return true; // nothing broken, there's just noone listening + + size_t outlen = strlen(outputbuf); + +#ifdef _WIN32 + for (unsigned i = 0; iclient ? maxfd : client); + FD_SET(client, &master); + _write(client, "HTTP/1.0 200 OK\r\n", 0); + _write(client, + "Server: Mozarella/2.2\r\n" + "Accept-Range: bytes\r\n" + "Connection: close\r\n" + "Max-Age: 0\r\n" + "Expires: 0\r\n" + "Cache-Control: no-cache, private\r\n" + "Pragma: no-cache\r\n" + "Content-Type: application/json\r\n" + //"Content-Type: multipart/x-mixed-replace; boundary=boundary\r\n" + "\r\n", 0); + cerr << "new client " << client << endl; + } + else // existing client, just stream pix + { + char head[400]; + // application/json + // application/x-resource+json or application/x-collection+json - when you are representing REST resources and collections + // text/json or text/javascript or text/plain. + // https://stackoverflow.com/questions/477816/what-is-the-correct-json-content-type + //sprintf(head, "\r\nContent-Length: %zu\r\n\r\n", outlen); + //sprintf(head, "--boundary\r\nContent-Type: application/json\r\nContent-Length: %zu\r\n\r\n", outlen); + _write(s, head, 0); + int n = _write(s, outputbuf, outlen); + //cerr << "known client " << s << " " << n << endl; + if (n < outlen) + { + cerr << "kill client " << s << endl; + ::shutdown(s, 2); + FD_CLR(s, &master); + } + } + } + return true; + } +}; +// ---------------------------------------- + +// JSON format: +//{ +// "frame_id":8990, +// "objects":[ +// {"class_id":4, "name":"aeroplane", "relative coordinates":{"center_x":0.398831, "center_y":0.630203, "width":0.057455, "height":0.020396}, "confidence":0.793070}, +// {"class_id":14, "name":"bird", "relative coordinates":{"center_x":0.398831, "center_y":0.630203, "width":0.057455, "height":0.020396}, "confidence":0.265497} +// ] +//}, + +void send_json(detection *dets, int nboxes, int classes, char **names, long long int frame_id, int port, int timeout) +{ + static JSON_sender js(port, timeout); + char *send_buf = detection_to_json(dets, nboxes, classes, names, frame_id); + + js.write(send_buf); + std::cout << " JSON-stream sent. \n"; + free(send_buf); +} + +// ---------------------------------------- + CvCapture* get_capture_video_stream(char *path) { CvCapture* cap = NULL; try { diff --git a/src/http_stream.h b/src/http_stream.h index 04c7516b..7710920d 100644 --- a/src/http_stream.h +++ b/src/http_stream.h @@ -15,6 +15,7 @@ extern "C" { #include #ifdef OPENCV +void send_json(detection *dets, int nboxes, int classes, char **names, long long int frame_id, int port, int timeout); void send_mjpeg(IplImage* ipl, int port, int timeout, int quality); CvCapture* get_capture_webcam(int index); CvCapture* get_capture_video_stream(char *path); diff --git a/src/network.c b/src/network.c index 306918bb..c2a69579 100644 --- a/src/network.c +++ b/src/network.c @@ -658,6 +658,44 @@ void free_detections(detection *dets, int n) free(dets); } +char *detection_to_json(detection *dets, int nboxes, int classes, char **names, long long int frame_id) +{ + const float thresh = 0.005; // function get_network_boxes() has already filtred dets by actual threshold + + char *send_buf = (char *)calloc(1024, sizeof(char)); + sprintf(send_buf, "{\n \"frame_id\":%d, \n \"objects\":[ \n", frame_id); + + int i, j; + int class_id = -1; + for (i = 0; i < nboxes; ++i) { + for (j = 0; j < classes; ++j) { + int show = strncmp(names[j], "dont_show", 9); + if (dets[i].prob[j] > thresh && show) + { + if (class_id != -1) strcat(send_buf, ", \n"); + class_id = j; + char *buf = (char *)calloc(2048, sizeof(char)); + //sprintf(buf, "{\"image_id\":%d, \"category_id\":%d, \"bbox\":[%f, %f, %f, %f], \"score\":%f}", + // image_id, j, dets[i].bbox.x, dets[i].bbox.y, dets[i].bbox.w, dets[i].bbox.h, dets[i].prob[j]); + + sprintf(buf, " {\"class_id\":%d, \"name\":\"%s\", \"relative coordinates\":{\"center_x\":%f, \"center_y\":%f, \"width\":%f, \"height\":%f}, \"confidence\":%f}", + j, names[j], dets[i].bbox.x, dets[i].bbox.y, dets[i].bbox.w, dets[i].bbox.h, dets[i].prob[j]); + + int send_buf_len = strlen(send_buf); + int buf_len = strlen(buf); + int total_len = send_buf_len + buf_len + 100; + send_buf = (char *)realloc(send_buf, total_len * sizeof(char)); + if (!send_buf) return;// exit(-1); + strcat(send_buf, buf); + free(buf); + } + } + } + strcat(send_buf, "\n ] \n}, \n"); + return send_buf; +} + + float *network_predict_image(network *net, image im) { //image imr = letterbox_image(im, net->w, net->h); diff --git a/src/yolo.c b/src/yolo.c index 4d344f0d..916e5474 100644 --- a/src/yolo.c +++ b/src/yolo.c @@ -339,6 +339,7 @@ void run_yolo(int argc, char **argv) { int dont_show = find_arg(argc, argv, "-dont_show"); int http_stream_port = find_int_arg(argc, argv, "-http_port", -1); + int json_port = find_int_arg(argc, argv, "-json_port", -1); char *out_filename = find_char_arg(argc, argv, "-out_filename", 0); char *prefix = find_char_arg(argc, argv, "-prefix", 0); float thresh = find_float_arg(argc, argv, "-thresh", .2); @@ -359,5 +360,5 @@ void run_yolo(int argc, char **argv) else if(0==strcmp(argv[2], "valid")) validate_yolo(cfg, weights); else if(0==strcmp(argv[2], "recall")) validate_yolo_recall(cfg, weights); else if(0==strcmp(argv[2], "demo")) demo(cfg, weights, thresh, hier_thresh, cam_index, filename, voc_names, 20, frame_skip, - prefix, out_filename, http_stream_port, dont_show, ext_output); + prefix, out_filename, http_stream_port, json_port, dont_show, ext_output); }