Modifikace README, zadání 1.0

README.md

@@ -1 +1,10 @@
-Zadání projektu do předmětu pokročilé asemblery 2024
+# IPA projekt - Postprocessing dat z region proposal detekční neuronové sítě
+
+Zadání projektu do předmětu pokročilé asemblery pro akademický rok 2023/2024.
+
+# Struktura repozitáře
+* vs_linux - projekt pro VS Code pro OS linux
+* vs_win - projekt pro VS Code pro OS Windows. Nutné zkopírovat do kořenového adresáře šablony z druhého cvičení (https://www.fit.vutbr.cz/~igoldmann/courses/IPA/cv2_64bit_2021_win.zip). 
+* zadani_x_x.pdf (1.0) - zadání projektu
+
+Realizaci můžete dělat s využitím intrinsic funkcí tak přímo v asembleru.

vs_linux/cv2_student_nasm.s

@@ -1,15 +0,0 @@
-#Not used for project
-
-[BITS 64]
-
-global f1:function
-
-
-section .data
-
-
-section .text
-
-f1:
-mov rax, 0
-ret

vs_linux/include/prior_boxes.hpp

@@ -1,3 +1,9 @@
+/*
+* Prior boxes for RetinaNet
+* Tomas Goldmann,2023
+*/
+
+
 #include <iostream>
 #include <vector>
 #include <cmath>

vs_linux/include/reader.hpp

@@ -1,3 +1,8 @@
+/*
+* Functions for reading and processing txt file with floats
+* Tomas Goldmann,2024
+*/
+
 #pragma once
 
 #include <iostream>

vs_linux/include/utils.hpp

@@ -1,3 +1,9 @@
+/*
+* Non-maximum suppression
+* Tomas Goldmann,2024
+*/
+
+
 #include <iostream>
 #include <vector>
 #include <cmath>

vs_linux/retinanetpost.cpp

@@ -3,7 +3,6 @@
 #include <string>
 #include <iostream>
 #include <inttypes.h>
-#include <dlfcn.h>
 #include "ipa_tool.h"
 
 #include <opencv2/highgui.hpp>
@@ -17,8 +16,13 @@ using namespace cv;
 #include "utils.hpp"
 #include "reader.hpp"
 
+//example: call extern function
 extern "C" { void f1(int a);}
 
+#define CONFIDENCE_THRESHOLD 0.999
+#define INPUT_WIDTH 640
+#define INPUT_HEIGHT 480
+#define ANCHORS_COUNT 12600
 
 int main(int argc, char** argv)
 {
@@ -31,21 +35,25 @@ int main(int argc, char** argv)
 
 	if (image.empty()) 
 	{
-        cout << "Could not open or find the image" << endl;
+        cout << "Could not open or find a image" << endl;
         return -1;
     }
 
 	//All constants refer to the configuration used in prior_boxes.cpp and to the 640x480 resolution.
-	std::vector<int> image_size = {640, 480};
+	std::vector<int> image_size = {INPUT_WIDTH, INPUT_HEIGHT};
 	std::vector<float> variances = {0.1f, 0.2f};
-	size_t total0_len = 12600*4;
+	size_t total0_len = ANCHORS_COUNT*4;
-	size_t total1_len = 12600*2;
+	size_t total1_len = ANCHORS_COUNT*2;
 	size_t num_anchors = total0_len / 4;
 
 	PriorBox priorBox(image_size, "projekt");
 
     std::vector<std::vector<float>>  priors = priorBox.forward();
 
+
+    Scalar color(0, 255, 0); // Color of the rectangle (in BGR)
+    int thickness = 2; // Thickness of the rectangle border
+
 	InstructionCounter counter;
 	counter.start();
 	/*******************Part to optmize*********************/
@@ -72,7 +80,7 @@ int main(int argc, char** argv)
 
     for (size_t i = 0; i < total1_len/2; i++) {
         scores.push_back(floatarrscr[i*2+1]);
-        if(floatarrscr[i*2+1] > 0.999)
+        if(floatarrscr[i*2+1] > CONFIDENCE_THRESHOLD)
         {
             inds.push_back(i);
             decoded_boxes[i].push_back(floatarrscr[i*2+1]);
@@ -87,17 +95,14 @@ int main(int argc, char** argv)
 
     auto out = nms(det_boxes, 0.4);
 
-	f1(10);
+    //Test
+	//f1(10);
 
 	counter.print();
 
-
 	/************************************************/
 
 
-    Scalar color(0, 255, 0); // Color of the rectangle (in BGR)
-    int thickness = 2; // Thickness of the rectangle border
-
     for (int i = 0; i < out.size(); i++)
     {
 		#ifdef DEBUG

vs_linux/retinanetpost_asm.s

@@ -1,7 +1,6 @@
 .intel_syntax noprefix
 
 .data
-	constants: .byte 50,50,50,50
 
 .text
 

vs_win/.vscode/c_cpp_properties.json

@@ -0,0 +1,21 @@
+{
+    "configurations": [
+        {
+            "name": "Win32",
+            "includePath": [
+                "${workspaceFolder}/**",
+                "${workspaceFolder}\\..\\include"
+            ],
+            "defines": [
+                "_DEBUG",
+                "UNICODE",
+                "_UNICODE"
+            ],
+            "compilerPath": "${workspaceFolder}\\..\\mingw64\\bin\\g++.exe",
+            "cStandard": "c11",
+            "cppStandard": "c++17",
+            "intelliSenseMode": "gcc-x64"
+        }
+    ],
+    "version": 4
+}

vs_win/.vscode/launch.json

@@ -0,0 +1,40 @@
+{
+    "version": "0.2.0",
+    "configurations": [
+      {
+        "name": "(gdb) Launch",
+        "type": "cppdbg",
+        "request": "launch",
+        "program": "${workspaceFolder}/retinanetpost.exe",
+        "args": ["input/input.jpg", "input/input.txt"],
+        "stopAtEntry": true,
+        "cwd": "${workspaceFolder}",
+
+        "environment": [{"name": "PATH", "value": "${env:PATH};${workspaceFolder}/runtime_lib/"}],
+        "externalConsole": false,
+        "preLaunchTask": "Build IPA_projekt_2024",
+        "windows": {
+          "MIMode": "gdb",
+          "miDebuggerPath": "${workspaceFolder}/../mingw64/bin/gdb.exe",
+          "setupCommands": [
+            {
+              "description": "Enable pretty-printing for gdb",
+              "text": "-enable-pretty-printing",
+              "ignoreFailures": true
+            },
+            {
+              "description": "Reduce gdb verbosity",
+              "text": "set print thread-events on",
+              "ignoreFailures": true
+            }
+          ],
+          "logging": {
+            "trace": false,
+            "traceResponse": false,
+            "engineLogging": false
+          }
+        }
+      }
+    ]
+
+  }

vs_win/.vscode/settings.json

@@ -0,0 +1,5 @@
+{
+    "files.associations": {
+        "iostream": "cpp"
+    }
+}

vs_win/.vscode/tasks.json

@@ -0,0 +1,84 @@
+{
+    "version": "2.0.0",
+    "windows": {
+
+    },
+    "tasks": [
+        {
+            "label": "Build IPA_projekt_2024",
+            "type": "shell",
+            "command": "g++",
+            "args": [
+                "-g",
+                "-std=c++11",
+                "-fpic",
+                "-o",
+                "retinanetpost.exe",
+                "-DDEBUG", 
+                "retinanetpost_asm.o",
+                "retinanetpost.cpp",
+                "src/utils.cpp",
+                "src/prior_boxes.cpp",
+                "src/reader.cpp",
+                "-Iinclude",
+                "-L..\\lib",
+                "-I..\\include",
+                "-lopencv_world451"
+            ],
+            "group": {
+                "kind": "build",
+                "isDefault": true
+            },
+            "problemMatcher": [],
+            "dependsOn": [
+                "ASM Part"
+            ],
+            "isBackground": false
+        },
+        {
+            "label": "ASM Part",
+            "type": "shell",
+            "command": "gcc",
+            "args": [
+                "-masm=intel",
+                "-g",
+                "-c",
+                "retinanetpost_asm.s"
+            ],
+            "group": {
+                "kind": "build",
+                "isDefault": true
+            },
+            "problemMatcher": [],
+            "dependsOn": [
+
+            ],
+            "isBackground": false
+        },
+        {
+            "label": "Run",
+            "type": "shell",
+            "command": "retinanetpost.exe",
+            "args": [
+                "input/input.jpg",
+
+                "input/input.txt"
+            ],
+            "group": {
+                "kind": "build",
+                "isDefault": true
+            },
+            "options": {
+                "env": {
+                    "PATH": "${env:PATH};${workspaceFolder}/runtime_lib/"
+                }
+            },
+            
+            "problemMatcher": [],
+            "dependsOn": [
+                "Build IPA_projekt_2024"
+            ],
+            "isBackground": false
+        }
+    ]
+  }

vs_win/include/prior_boxes.hpp

@@ -0,0 +1,29 @@
+/*
+* Prior boxes for RetinaNet
+* Tomas Goldmann,2023
+*/
+
+
+#include <iostream>
+#include <vector>
+#include <cmath>
+
+class PriorBox {
+
+
+private:
+    std::vector<std::vector<int>> feature_maps;
+    std::vector<std::vector<float>> min_sizes;
+    std::vector<int> steps;
+    bool clip;
+    std::vector<int> image_size;
+    std::string name;
+
+
+public:
+    PriorBox(std::vector<int> image_size = std::vector<int>(), std::string phase = "train");
+    std::vector<std::vector<float>> forward();
+
+};
+
+std::vector<std::vector<float>> decode(const std::vector<std::vector<float>>& loc, const std::vector<std::vector<float>>& priors, const std::vector<float>& variances);

vs_win/include/reader.hpp

@@ -0,0 +1,18 @@
+/*
+* Functions for reading and processing txt file with floats
+* Tomas Goldmann,2024
+*/
+
+
+#pragma once
+
+#include <iostream>
+#include <fstream>
+#include <sstream>
+#include <vector>
+
+using namespace std;
+
+
+vector<vector<float>> splitFloats(const vector<float>& floats, int index) ;
+vector<float> readFloatsFromFile(const string& filename);

vs_win/include/utils.hpp

@@ -0,0 +1,13 @@
+/*
+* Non-maximum suppression
+* Tomas Goldmann,2024
+*/
+
+
+
+#include <iostream>
+#include <vector>
+#include <cmath>
+#include <algorithm>
+
+std::vector<std::vector<float>> nms(std::vector<std::vector<float>>& bboxes, float threshold);

vs_win/ipa_tool.h

@@ -0,0 +1,35 @@
+#include <stdio.h>
+#include <inttypes.h>
+#include <x86intrin.h>
+
+
+class InstructionCounter
+{
+private:
+	unsigned  long long   counter;
+	unsigned  long long  end_count;
+	unsigned  long long total_count;
+public:
+	
+	void start()
+	{
+		counter= __rdtsc();
+	};
+
+	void end()
+	{
+		end_count = __rdtsc();
+	};
+
+	void print()
+	{
+		printf("%I64lld \n", getCyclesCount());
+	}
+
+	unsigned long long  getCyclesCount()
+	{
+		end();
+		total_count = end_count - counter;
+		return  total_count;
+	}
+};

vs_win/retinanetpost.cpp

@@ -0,0 +1,121 @@
+
+#include <cstring>
+#include <string>
+#include <iostream>
+#include <inttypes.h>
+#include "ipa_tool.h"
+
+#include <opencv2/highgui.hpp>
+#include <opencv2/core.hpp>
+#include <opencv2/imgproc.hpp>
+
+using namespace std;
+using namespace cv;
+
+#include "prior_boxes.hpp"
+#include "utils.hpp"
+#include "reader.hpp"
+
+//example: call extern function
+extern "C" { void f1(int a);}
+
+#define CONFIDENCE_THRESHOLD 0.999
+#define INPUT_WIDTH 640
+#define INPUT_HEIGHT 480
+#define ANCHORS_COUNT 12600
+
+int main(int argc, char** argv)
+{
+	if (argc != 3)
+	{
+		std::cout << "Run program by: ./retinapost input/vector.txt input/image.png";
+	}
+
+	Mat image = imread(argv[1]);
+
+	if (image.empty()) 
+	{
+        cout << "Could not open or find a image" << endl;
+        return -1;
+    }
+
+	//All constants refer to the configuration used in prior_boxes.cpp and to the 640x480 resolution.
+	std::vector<int> image_size = {INPUT_WIDTH, INPUT_HEIGHT};
+	std::vector<float> variances = {0.1f, 0.2f};
+	size_t total0_len = ANCHORS_COUNT*4;
+	size_t total1_len = ANCHORS_COUNT*2;
+	size_t num_anchors = total0_len / 4;
+
+	PriorBox priorBox(image_size, "projekt");
+
+    std::vector<std::vector<float>>  priors = priorBox.forward();
+
+
+    Scalar color(0, 255, 0); // Color of the rectangle (in BGR)
+    int thickness = 2; // Thickness of the rectangle border
+
+	InstructionCounter counter;
+	counter.start();
+	/*******************Part to optmize*********************/
+
+	vector<float> floats = readFloatsFromFile(argv[2]);
+	vector<vector<float>> split_vectors = splitFloats(floats, 12600*4);
+
+	vector<float> floatarr = split_vectors[0];
+    vector<float> floatarrscr = split_vectors[1];
+
+
+    std::vector<std::vector<float>> loc;
+    for (size_t i = 0; i < num_anchors; i++) {
+        loc.push_back({floatarr[i * 4], floatarr[i * 4 + 1], floatarr[i * 4 + 2], floatarr[i * 4 + 3]});
+    }
+
+    std::vector<std::vector<float>> decoded_boxes = decode(loc, priors, variances);
+
+    std::vector<float> scores;
+    std::vector<int> inds;
+    std::vector<float> det_scores;
+
+    std::vector<std::vector<float>> det_boxes;
+
+    for (size_t i = 0; i < total1_len/2; i++) {
+        scores.push_back(floatarrscr[i*2+1]);
+        if(floatarrscr[i*2+1] > CONFIDENCE_THRESHOLD)
+        {
+            inds.push_back(i);
+            decoded_boxes[i].push_back(floatarrscr[i*2+1]);
+            decoded_boxes[i][0]= decoded_boxes[i][0]* 640;
+            decoded_boxes[i][1]= decoded_boxes[i][1]* 480;
+            decoded_boxes[i][2]= decoded_boxes[i][2]* 640;
+            decoded_boxes[i][3]= decoded_boxes[i][3]* 480;
+            det_boxes.push_back(decoded_boxes[i]);
+            det_scores.push_back(scores[i]);
+        }
+    }
+
+    auto out = nms(det_boxes, 0.4);
+
+    //Test
+	//f1(10);
+
+	counter.print();
+
+	/************************************************/
+
+
+    for (int i = 0; i < out.size(); i++)
+    {
+		#ifdef DEBUG
+			printf("Box %f %f %f %f %f\n", out[i][0],  out[i][1],  out[i][2],  out[i][3],  out[i][4]);
+		#endif
+
+
+        cv::Rect roi((int)out[i][0],  (int)out[i][1],  (int)out[i][2]- (int)out[i][0], (int)out[i][3] - (int)out[i][1]);
+		rectangle(image, roi, color, thickness);
+    }
+
+    imshow("Output", image);
+    waitKey(0);
+
+	return 0;
+}

vs_win/retinanetpost_asm.s

@@ -0,0 +1,12 @@
+.intel_syntax noprefix
+
+.data
+
+.text
+
+.global f1
+f1:
+
+ret 0
+
+

vs_win/src/prior_boxes.cpp

@@ -0,0 +1,109 @@
+/*
+* Prior boxes for RetinaNet
+* Tomas Goldmann,2023
+*/
+
+#include <iostream>
+#include <vector>
+#include <cmath>
+#include <map>
+#include <algorithm>
+#include "prior_boxes.hpp"
+
+PriorBox::PriorBox(std::vector<int> image_size, std::string phase)
+{
+
+    std::map<std::string, std::vector<std::vector<float>>> cfg;
+    this->min_sizes = {{16.0f, 32.0f}, {64.0f, 128.0f}, {256.0f, 512.0f}};
+    this->image_size = {image_size[1], image_size[0]};
+    this->steps = {8, 16, 32};
+
+    for (const float step : steps)
+    {
+        this->feature_maps.push_back({(int)std::ceil(image_size[0] / step), (int)std::ceil(image_size[1] / step)});
+    }
+}
+
+std::vector<std::vector<float>> PriorBox::forward()
+{
+
+    std::vector<float> anchors;
+    for (size_t k = 0; k < this->feature_maps.size(); k++)
+    {
+        const auto &f = this->feature_maps[k];
+        const auto &min_sizes = this->min_sizes[k];
+
+        for (int i = 0; i < f[1]; i++)
+        {
+            for (int j = 0; j < f[0]; j++)
+            {
+                for (const auto &min_size : min_sizes)
+                {
+                    float s_kx = min_size / this->image_size[1];
+                    float s_ky = min_size / this->image_size[0];
+                    std::vector<float> dense_cx = {static_cast<float>(j + 0.5) * this->steps[k] / this->image_size[1]};
+                    std::vector<float> dense_cy = {static_cast<float>(i + 0.5) * this->steps[k] / this->image_size[0]};
+
+                    for (const auto &cy : dense_cy)
+                    {
+                        for (const auto &cx : dense_cx)
+                        {
+
+                            anchors.push_back(cx);
+                            anchors.push_back(cy);
+                            anchors.push_back(s_kx);
+                            anchors.push_back(s_ky);
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    std::vector<std::vector<float>> output;
+
+    size_t num_anchors = anchors.size() / 4;
+    for (size_t i = 0; i < num_anchors; i++)
+    {
+        for (size_t j = 0; j < 4; j++)
+        {
+            anchors[i * 4 + j] = std::min(std::max(anchors[i * 4 + j], 0.0f), 1.0f);
+        }
+        output.push_back({anchors[i * 4], anchors[i * 4 + 1], anchors[i * 4 + 2], anchors[i * 4 + 3]});
+    }
+    //std::cout << "len  " << output.size() << std::endl;
+
+    return output;
+}
+
+std::vector<std::vector<float>> decode(const std::vector<std::vector<float>> &loc, const std::vector<std::vector<float>> &priors, const std::vector<float> &variances)
+{
+    std::vector<std::vector<float>> boxes;
+    for (size_t i = 0; i < loc.size(); i++)
+    {
+        const auto &prior = priors[i];
+        const auto &loc_pred = loc[i];
+        float prior_x = prior[0];
+        float prior_y = prior[1];
+        float prior_w = prior[2];
+        float prior_h = prior[3];
+        float var_x = variances[0];
+        float var_y = variances[1];
+        float var_w = variances[2];
+        float var_h = variances[3];
+
+        float decoded_x = prior_x + loc_pred[0] * var_x * prior_w;
+        float decoded_y = prior_y + loc_pred[1] * var_y * prior_h;
+        float decoded_w = prior_w * std::exp(loc_pred[2] * var_w);
+        float decoded_h = prior_h * std::exp(loc_pred[3] * var_h);
+
+        float decoded_xmin = decoded_x - decoded_w / 2;
+        float decoded_ymin = decoded_y - decoded_h / 2;
+        float decoded_xmax = decoded_x + decoded_w / 2;
+        float decoded_ymax = decoded_y + decoded_h / 2;
+
+        boxes.push_back({decoded_xmin, decoded_ymin, decoded_xmax, decoded_ymax});
+    }
+
+    return boxes;
+}

vs_win/src/reader.cpp

@@ -0,0 +1,59 @@
+/*
+* Function to read floats from txt
+* Tomas Goldmann,2023
+*/
+
+
+#include "reader.hpp"
+
+using namespace std;
+
+vector<vector<float>> splitFloats(const vector<float>& floats, int index) {
+
+    vector<float> first_half;
+    vector<float> second_half;
+
+    // Ensure the original vector has enough elements for splitting
+    if (floats.size() < 6*12600) {
+        cerr << "Error: The input must containt 6*12600 floats" << endl;
+        return {first_half, second_half}; // Return empty vectors
+    }
+
+    // Split the original vector into two separate vectors
+    first_half.insert(first_half.end(), floats.begin(), floats.begin() + index);
+    second_half.insert(second_half.end(), floats.begin() + index,  floats.end() );
+
+    // Return a vector containing the two split vectors
+    return {first_half, second_half};
+}
+
+
+vector<float> readFloatsFromFile(const string& filename) {
+    vector<float> floats;
+    ifstream file(filename);
+
+    // Check if the file is opened successfully
+    if (!file.is_open()) {
+        cerr << "Error opening the file" << endl;
+        return floats; // Return an empty vector if the file cannot be opened
+    }
+
+    string line;
+
+    // Read each line from the file
+    while (getline(file, line)) {
+        // Create a string stream from the line
+        stringstream ss(line);
+        string token;
+
+        while (getline(ss, token, ',')) {
+            // Convert the string to a float and store it in the vector
+            floats.push_back(stof(token));
+        }
+    }
+
+    // Close the file
+    file.close();
+
+    return floats;
+}

vs_win/src/utils.cpp

@@ -0,0 +1,62 @@
+#include "utils.hpp"
+
+
+bool CompareBBox(const std::vector<float> & a, const std::vector<float> & b)
+{
+    return a[4] > b[4];
+}
+
+std::vector<std::vector<float>> nms(std::vector<std::vector<float>>& bboxes, float threshold)
+{
+    std::vector<std::vector<float>> bboxes_nms;
+    std::sort(bboxes.begin(), bboxes.end(), CompareBBox);
+
+    int32_t select_idx = 0;
+    int32_t num_bbox = static_cast<int32_t>(bboxes.size());
+    std::vector<int32_t> mask_merged(num_bbox, 0);
+    bool all_merged = false;
+
+    while (!all_merged) {
+        while (select_idx < num_bbox && mask_merged[select_idx] == 1)
+            select_idx++;
+        if (select_idx == num_bbox) {
+            all_merged = true;
+            continue;
+        }
+
+        bboxes_nms.push_back(bboxes[select_idx]);
+        mask_merged[select_idx] = 1;
+
+        std::vector<float> select_bbox = bboxes[select_idx];
+        float area1 = static_cast<float>((select_bbox[2] - select_bbox[0] + 1) * (select_bbox[3] - select_bbox[1] + 1));
+        float x1 = static_cast<float>(select_bbox[0]);
+        float y1 = static_cast<float>(select_bbox[1]);
+        float x2 = static_cast<float>(select_bbox[2]);
+        float y2 = static_cast<float>(select_bbox[3]);
+
+        select_idx++;
+        for (int32_t i = select_idx; i < num_bbox; i++) {
+            if (mask_merged[i] == 1)
+                continue;
+
+            std::vector<float>& bbox_i = bboxes[i];
+            float x = std::max<float>(x1, static_cast<float>(bbox_i[0]));
+            float y = std::max<float>(y1, static_cast<float>(bbox_i[1]));
+            float w = std::min<float>(x2, static_cast<float>(bbox_i[2])) - x + 1;
+            float h = std::min<float>(y2, static_cast<float>(bbox_i[3])) - y + 1;
+            
+            if (w <= 0 || h <= 0)
+                continue;
+
+            float area2 = static_cast<float>((bbox_i[2] - bbox_i[0] + 1) * (bbox_i[3] - bbox_i[1] + 1));
+            float area_intersect = w * h;
+
+   
+            if (static_cast<float>(area_intersect) / (area1 + area2 - area_intersect) > threshold) {
+                mask_merged[i] = 1;
+            }
+        }
+    }
+
+    return bboxes_nms;
+}