3. OpenCV with petalinux

3.1. OpenCV Basics

3.1.1. Create Petalinux project

(activate the petalinux)
$ ts -petalinux 2018.2

(download the bsp first, and then use the below command)
$ petalinux-create -t project -n opencvExamples -s /proj/css/meherp/bsp/v2018_2/xilinx-zcu104-v2018.2-final.bsp

$ cd opencvExamples/


$ petalinux-config -c rootfs
    (select following options-> save -> exit)

    Filesystem Packages ---> libs ---> libmali-xlnx ---> [*] libmali-xlnx

    Petalinux Package Groups ---> packagegroup-petalinux-display-debug ---> [*] packagegroup-petalinux-display-debug

    Petalinux Package Groups ---> packagegroup-petalinux-opencv ---> [*] packagegroup-petalinux-opencv

    Petalinux Package Groups ---> packagegroup-petalinux-v4lutils ---> [*] packagegroup-petalinux-v4lutils

    Petalinux Package Groups ---> packagegroup-petalinux-x11 ---> [*] packagegroup-petalinux-x11


$ petalinux-build


(Next, create a petalinux application as below)
$ petalinux-create -t apps -n ocvtest --enable
  • Above will create a default ‘hello world’ app. Now, we need to modify the code for rotating the image by 90 degree. Modify the below files (vi editor is used below)

Note

If there are some error during petalinux-build then recreated the project and remove below line. Then run the petlainux build command,

do_compile() {
     oe_runmake
}
$ vi project-spec/meta-user/recipes-apps/ocvtest/ocvtest.bb
# (replace the code with below code)

#
# This file is the ocvtest recipe.
#

SUMMARY = "Simple ocvtest application"
SECTION = "PETALINUX/apps"
LICENSE = "MIT"
LIC_FILES_CHKSUM = "file://${COMMON_LICENSE_DIR}/MIT;md5=0835ade698e0bcf8506ecda2f7b4f302"

SRC_URI = "file://ocvtest.cpp \
           file://CMakeLists.txt \
          "

S = "${WORKDIR}"

DEPENDS += "opencv"

inherit pkgconfig cmake

do_compile() {
         oe_runmake
}

do_install() {
         install -d ${D}${bindir}
         install -m 0755 ocvtest ${D}${bindir}
}
  • Modify CMakeLists.txt
$ vi project-spec/meta-user/recipes-apps/ocvtest/files/CMakeLists.txt
# (replace the code with below code)


# cmake needs this line
cmake_minimum_required(VERSION 2.8)

add_definitions(-std=c++11 -Werror=return-type)

# Define project name
project(ocvtest)

# Find OpenCV, you may need to set OpenCV_DIR variable
# to the absolute path to the directory containing OpenCVConfig.cmake file
# via the command line or GUI
find_package(OpenCV REQUIRED)

# If the package has been found, several variables will
# be set, you can find the full list with descriptions
# in the OpenCVConfig.cmake file.
# Print some message showing some of them
message(STATUS "OpenCV library status:")
message(STATUS "    version: ${OpenCV_VERSION}")
message(STATUS "    libraries: ${OpenCV_LIBS}")
message(STATUS "    include path: ${OpenCV_INCLUDE_DIRS}")

if(CMAKE_VERSION VERSION_LESS "2.8.11")
  # Add OpenCV headers location to your include paths
  include_directories(${OpenCV_INCLUDE_DIRS})
endif()


# Declare the executable target built from your sources
add_executable(ocvtest ocvtest.cpp)

# Link your application with OpenCV libraries
target_link_libraries(ocvtest ${OpenCV_LIBS})
  • Modify makefile
$ vi project-spec/meta-user/recipes-apps/ocvtest/files/Makefile
# replace the code with below code,



APP = ocvtest

# Add any other object files to this list below
APP_OBJS = ocvtest.o

all: build

build: $(APP)

CXXFLAGS += $(shell pkg-config --cflags opencv)
LDFLAGS += $(shell pkg-config --libs opencv)

$(APP): $(APP_OBJS)
    $(CXX) $(CXXFLAGS) $(LDFLAGS) -o $@ $(APP_OBJS) $(LDLIBS)

clean:
    rm -f $(APP) *.elf *.gdb *.o

3.1.2. Write OpenCV code

Below OpenCV code is exactly same as ‘OpenCV code for C++ i.e. bitwise_img.cpp’.

  • Modify ocvtest.cpp
// ocvtest.cpp

#include <stdio.h>
#include <opencv2/opencv.hpp>

using namespace std;

int main(int argc, char** argv ){

    cv::Mat and_img;

    // create and display frame of size 300 for rectangle and circle
    cv::Mat rectangle(300, 300, CV_8UC1, cv::Scalar(0)); // rectangle
    cv::Mat circle(300, 300, CV_8UC1, cv::Scalar(0)); // circle


    // draw and show rectangle
    cv::rectangle( rectangle, cv::Point(20, 20), cv::Point(280, 280), cv::Scalar( 255 ), -1);
    cv::imshow("Rectangle", rectangle);

    // draw and show circle
    cv::circle(circle, cv::Point(150, 150), 150, cv::Scalar(255), -1); // black
    cv::imshow("Circle", circle);

    // bitwise and operation
    cv::bitwise_and(circle, rectangle, and_img);
    cv::imshow("And", and_img);



    // another example
    cv::Mat rect1 = cv::Mat::zeros( cv::Size(400,200), CV_8UC1);
    cv::Mat rect2 = cv::Mat::zeros( cv::Size(400,200), CV_8UC1);


    rect1( cv::Range(0, 200), cv::Range(0, 200) ) = 255;
    cv::imshow("rect1", rect1);
    cv::imwrite("/home/root/rect1.jpg", rect1);
    cout << "Image written at /home/root/rect1.jpg " << endl;

    rect2( cv::Range(100, 150), cv::Range(150, 250) ) = 255;
    cv::imwrite("/home/root/rect2.jpg", rect2);
    cout << "Image written at /home/root/rect2.jpg " << endl;
    cv::imshow("rect2", rect2);

    cv::Mat result;

    bitwise_and(rect1, rect2, result);
    cv::imwrite("/home/root/result_and.jpg", result);
    cv::imshow("AND", result);

    bitwise_or(rect1, rect2, result);
    cv::imwrite("/home/root/result_or.jpg", result);
    cv::imshow("OR", result);

    bitwise_xor(rect1, rect2, result);
    cv::imwrite("/home/root/result_xor.jpg", result);
    cv::imshow("XOR", result);

    bitwise_not(rect2, result);
    cv::imwrite("/home/root/not_rect2.jpg", result);
    cv::imshow("rect2 NOT", result);

    cout << "before waitkey" << endl;
    cv::waitKey(0);

    return 0;
}
  • Build the project,
$  petalinux-build -c ocvtest
$  petalinux-build

3.1.3. Run the desing on FPGA

  • copy image.up and BOOT.bin file in SD card; and boot the FPGA.
  • In the below commands, 190.122.11.229 is IP address of FPGA.
(check the IP address of FPGA)
$ ifconfig


(below will connet to FPGA, 190.122.11.229 will be shown by above command)
(run from windows machine)
ssh root@190.122.11.155

(run on fpga i.e. after running the above command)
root@xilinx-zcu104-2018_2:~# mount /dev/mmcblk0p1 /mnt/
root@xilinx-zcu104-2018_2:~# ocvtest
    Image written at /home/root/rect1.jpg
    Image written at /home/root/rect2.jpg
    before waitkey

root@xilinx-zcu104-2018_2:~# cp *.jpg /mnt/outimg/
root@xilinx-zcu104-2018_2:~# ls /mnt/outimg/
    not_rect2.jpg   rect2.jpg       result_or.jpg
    rect1.jpg       result_and.jpg  result_xor.jpg



(run windows machine i.e. copy the images from the folder-outimg to local machine)
(./meher is the location in windows harddisk)


(windows terminal)
cd C:
scp root@190.122.11.155:/mnt/outimg/*.jpg ./meher/outputs
../_images/Screenshot_121.jpg

Fig. 3.1 Bitwise operation

3.1.4. Drawing

  • In the above code, we did not read the image in the C++ code (we created the square and rectangle using commands only).
  • We need to modify the code slightly to read the images from the SD card. In the other word, we need to provide the location of the image manually which requires the ‘argv’ in main function. Note that, the same code can be used with OpenCV C++ as well.
  • In the below code, ocvtest.cpp is modified for drawing images. The C++ code ‘drawing_img.cpp’ is slightly modified to read the images (see highlighted section); and rest of the code is same.
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
// ocvtest.cpp

#include <stdio.h>
#include <opencv2/opencv.hpp>

using namespace std;

int main(int argc, char **argv)
{
    if( argc != 2)
    {
        cout <<" Usage: ./ocvtest <image-name>.jpg" << std::endl;
        return -1;
    }

    // read image
    cv::Mat img = cv::imread(argv[1]);

    // Check for invalid input
    if( img.empty() )
    {
        cout <<  "Could not open or find the image" << std::endl ;
        return -1;
    }

    // img = cv::imread("images/shapes.jpg");
    cv::imshow("Shapes", img);

    int w, h; // width, height
    // width and height of image
    w = img.size().width;
    h = img.size().height;

    // draw horizontal line
    cv::line( img, cv::Point( 0, (int)h/2 ), cv::Point( w, (int)h/2), cv::Scalar( 255, 0, 0 ), 3);
    // draw vertical line
    cv::line( img, cv::Point( (int)w/2, 0 ), cv::Point( (int)w/2, h), cv::Scalar( 255, 0, 0 ), 3);

    // draw rectangle
    cv::rectangle( img, cv::Point(5, 10), cv::Point(200, 170), cv::Scalar( 0, 0, 255 ), 3);

    // draw circle
    // center coordinates (w//2, h//2) and radius (50) are
    // required to to draw circle. 10 is the line width
    cv::circle(img, cv::Point((int)w/2, (int)h/2), 50, cv::Scalar(0, 0, 0), 10); // black
    cv::circle(img, cv::Point((int)w/2, (int)h/2), 30, cv::Scalar(0, 0, 255), -1); // -1 : filled circle

    cv::imshow("Shapes", img);
    cv::imwrite("/home/root/drawing_img.jpg", img);
    cv::waitKey(0);
    return 0;
}
  • Build the project,
$  petalinux-build -c ocvtest
$  petalinux-build
  • Copy the image.ub and BOOT.bin file in SD-card and run the design on FPGA
ssh root@190.122.11.155
mount /dev/mmcblk0p1 /mnt/
ocvtest
    Usage: ./ocvtest <image-name>.jpg
ocvtest /mnt/images/shapes.jpg

cp /home/root/drawing_img.jpg /mnt/outimg/

(windows terminal)
cd C:
scp root@190.122.11.155:/mnt/outimg/*.jpg ./meher/outputs
../_images/Screenshot_111.jpg

Fig. 3.2 Drawing

3.1.5. Sobel and Canny Edge detection

In the below code, Sobel edge detection and Canny edge detection algorithm are implemented.

// ocvtest.cpp


#include <stdio.h>
#include <opencv2/opencv.hpp>

using namespace std;

int main(int argc, char **argv)
{
    if( argc != 2)
    {
        cout <<" Usage: ./rotateimage <image-name>.jpg" << std::endl;
        return -1;
    }

    // read image
    cv::Mat img = cv::imread(argv[1]);

    // Check for invalid input
    if( img.empty() )
    {
        cout <<  "Could not open or find the image" << std::endl ;
        return -1;
    }

    // img = cv::imread("images/shapes.jpg");

    cv::imshow("Lego color", img);

    int w, h;
    // width and height of image
    w = img.size().width;
    h = img.size().height;

    cv::Mat gray_img;
    cv::cvtColor(img, gray_img, cv::COLOR_BGR2GRAY);
    cv::imshow("Lego ", gray_img);


    // sobel edge detection
    cv::Mat gX, gY;
    //  compute gradients along the X and Y axis, respectively
    cv::Sobel(gray_img, gX, CV_64F, 1, 0);
    cv::Sobel(gray_img, gY, CV_64F, 0, 1);
    // gX value after sobel conversion -52.0
    cout << "gX value after sobel conversion: " << (int)gX.at<double>(100, 200) << endl;


    // gX and gY are decimal number with +/- values
    // change these values to +ve integer format
    cv::convertScaleAbs(gX, gX);
    // gX value after Absolute scaling 52
    cv::convertScaleAbs(gY, gY);
    cout << "gX value after Absolute scaling: " << (int)gX.at<uchar>(100, 200) << endl;


    cv::Mat sobelCombined;
    cv::addWeighted(gX, 0.5, gY, 0.5, 0, sobelCombined);

    // show the output images
    cv::imshow("Sobel X", gX);
    cv::imwrite("/home/root/Sobel X.jpg", gX);
    cv::imshow("Sobel Y", gY);
    cv::imwrite("/home/root/Sobel Y.jpg", gY);
    cv::imshow("Sobel Combined", sobelCombined);
    cv::imwrite("/home/root/Sobel Combined.jpg", sobelCombined);


    // Canny edge detection
    cv::Mat canny_wide, canny_medium, canny_narrow;

    cv::Canny(gray_img, canny_wide, 10, 200);
    cv::Canny(gray_img, canny_medium, 50, 150);
    cv::Canny(gray_img, canny_narrow, 200, 250);

    // show the output images
    cv::imshow("Canny (10, 200)", canny_wide);
    cv::imwrite("/home/root/canny_wide.jpg", canny_wide);
    cv::imshow("Canny (50, 150)", canny_medium);
    cv::imwrite("/home/root/canny_medium.jpg", canny_medium);
    cv::imshow("Canny (200, 250)", canny_narrow);
    cv::imwrite("/home/root/canny_narrow.jpg", canny_narrow);

    cv::waitKey(0);
    return 0;
}
  • Build the project,
$  petalinux-build -c ocvtest
$  petalinux-build
  • Copy the image.ub and BOOT.bin file in SD-card and run the design on FPGA
ssh root@190.122.11.155
mount /dev/mmcblk0p1 /mnt/
ocvtest /mnt/images/lego.jpg

cp /home/root/*.jpg /mnt/outimg/

(windows terminal)
cd C:
scp root@190.122.11.155:/mnt/outimg/*.jpg ./meher/outputs
  • Output figure for Sobel detection is shown Fig. 3.3,
../_images/Screenshot_141.jpg

Fig. 3.3 Sobel edge detection

  • Output figure for Canny detection is shown Fig. 3.4,
../_images/Screenshot_151.jpg

Fig. 3.4 Canny edge detection