Ok, here comes another source code running record ← ←
1. Basic environment
According to the author's source code Readme file, Python 2.7 is a must. Afterwards, I installed tensorflow and everything was done in a virtual environment established by Anaconda. This record is for the OpenCV 3.x version! The modification of the 3.x version is the fourth major point, and the front part is the general part.
2. Satisfy the conditions for compiling ORB-SLAM2
Because this source code is written based on ORB-SLAM2, of course it must meet some preconditions. It needs to pre-install some libraries such as C++11 or C++0x Compiler, Pangolin, OpenCV and Eigen3. If you don’t know how to install it, you can install it again Do a manual search. When DynaSLAM first came out, it could only support OpenCV2.4.11, but in 2019, some enthusiastic people submitted code that can support Opencv3.x, and we will talk about it in detail later. I installed OpenCV 3.4.5 myself
3. Other libraries that need to be installed before compiling DynaSLAM
According to the Readme file of the open source code:
1. Install the boost library
sudo apt-get install libboost-all-dev
2. Download the DynaSLAM source code and put it in the h5 file
git clone https://github.com/BertaBescos/DynaSLAM.git
Then download the h5 file from this page https://github.com/matterport/Mask_RCNN/releasesDynaSLAM/src/python/ and save the file to
3.Python related environment
Here first create a new virtual environment in Anaconda and activate it, and then install tensorflow and keras in the virtual environment in turn.
conda create -n MaskRCNN python=2.7
conda activate MaskRCNN
pip install tensorflow==1.14.0 #或者 pip install tensorflow-gpu==1.14.0
pip install keras==2.0.9
After completing the above steps, the python environment is almost ready, you can test it below
cd DynaSLAM
python src/python/Check.py
If the output is Mask R-CNN is correctly working, you can go to the next step. However, it is difficult for things to go so smoothly hahaha, so let's solve them one by one. I ran into two problems here:
3.1 scikit-image is not installed
sudo pip install scikit-image
3.2 Errors about pycocotools
Notice! It must be installed when Python2.7 (cocoapi only supports Python2)! Otherwise, when running Check.py, an error will be reported that _mask cannot be found, because the file _mask.so will not be generated if Python3 is running.
git clone https://github.com/waleedka/coco
python PythonAPI/setup.py build_ext install
After running the above command, copy the entire pycocotools folder to src/python/the next, like this:
4. Modify part of DynaSLAM source code
Thank you very much for this young lady (?), the original address is here: Pushyami_dev , if you want to see the specific additions and deletions of the code, you can click to see it.
The submitted code is mainly for the use of Opencv3, and then based on this code, remove -march=native in CMakeLists.txt (there will be a Segment Default error report), /Thirdparty/DBoW2and remember to go there. The code is mainly modified in the following parts, just copy it to the folder directly, pay attention to modifying your own version of OpenCV3.x.
- CMakeLists.txt
- Thirdparty/DBoW2/CMakeLists.txt
- include/Conversion.h
- src/Conversion.cc
1. CMakeLists.txt
cmake_minimum_required(VERSION 2.8)
project(DynaSLAM)
IF(NOT CMAKE_BUILD_TYPE)
SET(CMAKE_BUILD_TYPE Release)
# SET(CMAKE_BUILD_TYPE Debug)
ENDIF()
MESSAGE("Build type: " ${CMAKE_BUILD_TYPE})
#set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -O3 -march=native ")
#set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -O3 -march=native")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -O3 ")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -O3 ")
# This is required if opencv is built from source locally
#SET(OpenCV_DIR "~/opencv/build")
# set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -O0 -march=native ")
# set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -O0 -march=native")
# Check C++11 or C++0x support
include(CheckCXXCompilerFlag)
CHECK_CXX_COMPILER_FLAG("-std=c++11" COMPILER_SUPPORTS_CXX11)
CHECK_CXX_COMPILER_FLAG("-std=c++0x" COMPILER_SUPPORTS_CXX0X)
if(COMPILER_SUPPORTS_CXX11)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
add_definitions(-DCOMPILEDWITHC11)
message(STATUS "Using flag -std=c++11.")
elseif(COMPILER_SUPPORTS_CXX0X)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++0x")
add_definitions(-DCOMPILEDWITHC0X)
message(STATUS "Using flag -std=c++0x.")
else()
message(FATAL_ERROR "The compiler ${CMAKE_CXX_COMPILER} has no C++11 support. Please use a different C++ compiler.")
endif()
LIST(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake_modules)
set(Python_ADDITIONAL_VERSIONS "2.7")
#This is to avoid detecting python 3
find_package(PythonLibs 2.7 EXACT REQUIRED)
if (NOT PythonLibs_FOUND)
message(FATAL_ERROR "PYTHON LIBS not found.")
else()
message("PYTHON LIBS were found!")
message("PYTHON LIBS DIRECTORY: " ${PYTHON_LIBRARY} ${PYTHON_INCLUDE_DIRS})
endif()
message("PROJECT_SOURCE_DIR: " ${OpenCV_DIR})
find_package(OpenCV 3.4 QUIET)
if(NOT OpenCV_FOUND)
find_package(OpenCV 2.4 QUIET)
if(NOT OpenCV_FOUND)
message(FATAL_ERROR "OpenCV > 2.4.x not found.")
endif()
endif()
find_package(Qt5Widgets REQUIRED)
find_package(Qt5Concurrent REQUIRED)
find_package(Qt5OpenGL REQUIRED)
find_package(Qt5Test REQUIRED)
find_package(Boost REQUIRED COMPONENTS thread)
if(Boost_FOUND)
message("Boost was found!")
message("Boost Headers DIRECTORY: " ${Boost_INCLUDE_DIRS})
message("Boost LIBS DIRECTORY: " ${Boost_LIBRARY_DIRS})
message("Found Libraries: " ${Boost_LIBRARIES})
endif()
find_package(Eigen3 3.1.0 REQUIRED)
find_package(Pangolin REQUIRED)
set(PYTHON_INCLUDE_DIRS ${PYTHON_INCLUDE_DIRS} /usr/local/lib/python2.7/dist-packages/numpy/core/include/numpy)
include_directories(
${PROJECT_SOURCE_DIR}
${PROJECT_SOURCE_DIR}/include
${EIGEN3_INCLUDE_DIR}
${Pangolin_INCLUDE_DIRS}
${PYTHON_INCLUDE_DIRS}
/usr/include/python2.7/
#/usr/lib/python2.7/dist-packages/numpy/core/include/numpy/
${Boost_INCLUDE_DIRS}
)
message("PROJECT_SOURCE_DIR: " ${PROJECT_SOURCE_DIR})
set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${PROJECT_SOURCE_DIR}/lib)
add_library(${PROJECT_NAME} SHARED
src/System.cc
src/Tracking.cc
src/LocalMapping.cc
src/LoopClosing.cc
src/ORBextractor.cc
src/ORBmatcher.cc
src/FrameDrawer.cc
src/Converter.cc
src/MapPoint.cc
src/KeyFrame.cc
src/Map.cc
src/MapDrawer.cc
src/Optimizer.cc
src/PnPsolver.cc
src/Frame.cc
src/KeyFrameDatabase.cc
src/Sim3Solver.cc
src/Initializer.cc
src/Viewer.cc
src/Conversion.cc
src/MaskNet.cc
src/Geometry.cc
)
target_link_libraries(${PROJECT_NAME}
${OpenCV_LIBS}
${EIGEN3_LIBS}
${Pangolin_LIBRARIES}
${PROJECT_SOURCE_DIR}/Thirdparty/DBoW2/lib/libDBoW2.so
${PROJECT_SOURCE_DIR}/Thirdparty/g2o/lib/libg2o.so
/usr/lib/x86_64-linux-gnu/libpython2.7.so
${Boost_LIBRARIES}
)
# Build examples
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${PROJECT_SOURCE_DIR}/Examples/RGB-D)
add_executable(rgbd_tum
Examples/RGB-D/rgbd_tum.cc)
target_link_libraries(rgbd_tum ${PROJECT_NAME})
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${PROJECT_SOURCE_DIR}/Examples/Stereo)
add_executable(stereo_kitti
Examples/Stereo/stereo_kitti.cc)
target_link_libraries(stereo_kitti ${PROJECT_NAME})
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${PROJECT_SOURCE_DIR}/Examples/Monocular)
add_executable(mono_tum
Examples/Monocular/mono_tum.cc)
target_link_libraries(mono_tum ${PROJECT_NAME})
2.Thirdparty/DBoW2/CMakeLists.txt
cmake_minimum_required(VERSION 2.8)
project(DBoW2)
#set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -O3 -march=native ")
#set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -O3 -march=native")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -O3 ")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -O3 ")
set(HDRS_DBOW2
DBoW2/BowVector.h
DBoW2/FORB.h
DBoW2/FClass.h
DBoW2/FeatureVector.h
DBoW2/ScoringObject.h
DBoW2/TemplatedVocabulary.h)
set(SRCS_DBOW2
DBoW2/BowVector.cpp
DBoW2/FORB.cpp
DBoW2/FeatureVector.cpp
DBoW2/ScoringObject.cpp)
set(HDRS_DUTILS
DUtils/Random.h
DUtils/Timestamp.h)
set(SRCS_DUTILS
DUtils/Random.cpp
DUtils/Timestamp.cpp)
find_package(OpenCV 3.4 QUIET)
if(NOT OpenCV_FOUND)
find_package(OpenCV 2.4.3 QUIET)
if(NOT OpenCV_FOUND)
message(FATAL_ERROR "OpenCV > 2.4.3 not found.")
endif()
endif()
set(LIBRARY_OUTPUT_PATH ${PROJECT_SOURCE_DIR}/lib)
include_directories(${OpenCV_INCLUDE_DIRS})
add_library(DBoW2 SHARED ${SRCS_DBOW2} ${SRCS_DUTILS})
target_link_libraries(DBoW2 ${OpenCV_LIBS})
3.include/Conversion.h
/**
* This file is part of DynaSLAM.
* Copyright (C) 2018 Berta Bescos <bbescos at unizar dot es> (University of Zaragoza)
* For more information see <https://github.com/bertabescos/DynaSLAM>.
*
*/
#ifndef CONVERSION_H_
#define CONVERSION_H_
#include <Python.h>
#include <opencv2/imgproc/imgproc.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/core/core.hpp>
#include "numpy/ndarrayobject.h"
// #include "__multiarray_api.h"
#define NUMPY_IMPORT_ARRAY_RETVAL
namespace DynaSLAM
{
static PyObject* opencv_error = 0;
static int failmsg(const char *fmt, ...);
class PyAllowThreads;
class PyEnsureGIL;
#define ERRWRAP2(expr) \
try \
{
\
PyAllowThreads allowThreads; \
expr; \
} \
catch (const cv::Exception &e) \
{
\
PyErr_SetString(opencv_error, e.what()); \
return 0; \
}
static PyObject* failmsgp(const char *fmt, ...);
static size_t REFCOUNT_OFFSET = (size_t)&(((PyObject*)0)->ob_refcnt) +
(0x12345678 != *(const size_t*)"\x78\x56\x34\x12\0\0\0\0\0")*sizeof(int);
static inline PyObject* pyObjectFromRefcount(const int* refcount)
{
return (PyObject*)((size_t)refcount - REFCOUNT_OFFSET);
}
static inline int* refcountFromPyObject(const PyObject* obj)
{
return (int*)((size_t)obj + REFCOUNT_OFFSET);
}
class NumpyAllocator;
enum {
ARG_NONE = 0, ARG_MAT = 1, ARG_SCALAR = 2 };
class NDArrayConverter
{
private:
void init();
public:
NDArrayConverter();
//cv::Mat toMat(const PyObject* o); //issue bug
cv::Mat toMat(PyObject* o);
PyObject* toNDArray(const cv::Mat& mat);
};
}
#endif /* CONVERSION_H_ */
4. src/Conversion.cc
/**
* This file is part of DynaSLAM.
* Copyright (C) 2018 Berta Bescos <bbescos at unizar dot es> (University of Zaragoza)
* For more information see <https://github.com/bertabescos/DynaSLAM>.
*
*/
#include "Conversion.h"
#include <iostream>
namespace DynaSLAM
{
static void init()
{
import_array();
}
static int failmsg(const char *fmt, ...)
{
char str[1000];
va_list ap;
va_start(ap, fmt);
vsnprintf(str, sizeof(str), fmt, ap);
va_end(ap);
PyErr_SetString(PyExc_TypeError, str);
return 0;
}
class PyAllowThreads
{
public:
PyAllowThreads() : _state(PyEval_SaveThread()) {
}
~PyAllowThreads()
{
PyEval_RestoreThread(_state);
}
private:
PyThreadState* _state;
};
class PyEnsureGIL
{
public:
PyEnsureGIL() : _state(PyGILState_Ensure()) {
}
~PyEnsureGIL()
{
//std::cout << "releasing"<< std::endl;
PyGILState_Release(_state);
}
private:
PyGILState_STATE _state;
};
using namespace cv;
static PyObject* failmsgp(const char *fmt, ...)
{
char str[1000];
va_list ap;
va_start(ap, fmt);
vsnprintf(str, sizeof(str), fmt, ap);
va_end(ap);
PyErr_SetString(PyExc_TypeError, str);
return 0;
}
class NumpyAllocator : public MatAllocator
{
public:
#if ( CV_MAJOR_VERSION < 3)
NumpyAllocator() {
}
~NumpyAllocator() {
}
void allocate(int dims, const int* sizes, int type, int*& refcount,
uchar*& datastart, uchar*& data, size_t* step)
{
//PyEnsureGIL gil;
int depth = CV_MAT_DEPTH(type);
int cn = CV_MAT_CN(type);
const int f = (int)(sizeof(size_t)/8);
int typenum = depth == CV_8U ? NPY_UBYTE : depth == CV_8S ? NPY_BYTE :
depth == CV_16U ? NPY_USHORT : depth == CV_16S ? NPY_SHORT :
depth == CV_32S ? NPY_INT : depth == CV_32F ? NPY_FLOAT :
depth == CV_64F ? NPY_DOUBLE : f*NPY_ULONGLONG + (f^1)*NPY_UINT;
int i;
npy_intp _sizes[CV_MAX_DIM+1];
for( i = 0; i < dims; i++ )
{
_sizes[i] = sizes[i];
}
if( cn > 1 )
{
_sizes[dims++] = cn;
}
PyObject* o = PyArray_SimpleNew(dims, _sizes, typenum);
if(!o)
{
CV_Error_(CV_StsError, ("The numpy array of typenum=%d, ndims=%d can not be created", typenum, dims));
}
refcount = refcountFromPyObject(o);
npy_intp* _strides = PyArray_STRIDES(o);
for( i = 0; i < dims - (cn > 1); i++ )
step[i] = (size_t)_strides[i];
datastart = data = (uchar*)PyArray_DATA(o);
}
void deallocate(int* refcount, uchar*, uchar*)
{
//PyEnsureGIL gil;
if( !refcount )
return;
PyObject* o = pyObjectFromRefcount(refcount);
Py_INCREF(o);
Py_DECREF(o);
}
#else
NumpyAllocator() {
stdAllocator = Mat::getStdAllocator();
}
~NumpyAllocator() {
}
UMatData* allocate(PyObject* o, int dims, const int* sizes, int type,
size_t* step) const {
UMatData* u = new UMatData(this);
u->data = u->origdata = (uchar*) PyArray_DATA((PyArrayObject*) o);
npy_intp* _strides = PyArray_STRIDES((PyArrayObject*) o);
for (int i = 0; i < dims - 1; i++)
step[i] = (size_t) _strides[i];
step[dims - 1] = CV_ELEM_SIZE(type);
u->size = sizes[0] * step[0];
u->userdata = o;
return u;
}
UMatData* allocate(int dims0, const int* sizes, int type, void* data,
size_t* step, int flags, UMatUsageFlags usageFlags) const {
if (data != 0) {
CV_Error(Error::StsAssert, "The data should normally be NULL!");
// probably this is safe to do in such extreme case
return stdAllocator->allocate(dims0, sizes, type, data, step, flags,
usageFlags);
}
PyEnsureGIL gil;
int depth = CV_MAT_DEPTH(type);
int cn = CV_MAT_CN(type);
const int f = (int) (sizeof(size_t) / 8);
int typenum =
depth == CV_8U ? NPY_UBYTE :
depth == CV_8S ? NPY_BYTE :
depth == CV_16U ? NPY_USHORT :
depth == CV_16S ? NPY_SHORT :
depth == CV_32S ? NPY_INT :
depth == CV_32F ? NPY_FLOAT :
depth == CV_64F ?
NPY_DOUBLE :
f * NPY_ULONGLONG + (f ^ 1) * NPY_UINT;
int i, dims = dims0;
cv::AutoBuffer<npy_intp> _sizes(dims + 1);
for (i = 0; i < dims; i++)
_sizes[i] = sizes[i];
if (cn > 1)
_sizes[dims++] = cn;
PyObject* o = PyArray_SimpleNew(dims, _sizes, typenum);
if (!o)
CV_Error_(Error::StsError,
("The numpy array of typenum=%d, ndims=%d can not be created", typenum, dims));
return allocate(o, dims0, sizes, type, step);
}
bool allocate(UMatData* u, int accessFlags,
UMatUsageFlags usageFlags) const {
return stdAllocator->allocate(u, accessFlags, usageFlags);
}
void deallocate(UMatData* u) const {
if (u) {
PyEnsureGIL gil;
PyObject* o = (PyObject*) u->userdata;
Py_XDECREF(o);
delete u;
}
}
const MatAllocator* stdAllocator;
#endif
};
NumpyAllocator g_numpyAllocator;
NDArrayConverter::NDArrayConverter() {
init(); }
void NDArrayConverter::init()
{
import_array();
}
cv::Mat NDArrayConverter::toMat( PyObject *o)
{
cv::Mat m;
if(!o || o == Py_None)
{
if( !m.data )
m.allocator = &g_numpyAllocator;
}
if( !PyArray_Check(o) )
{
failmsg("toMat: Object is not a numpy array");
}
int typenum = PyArray_TYPE(o);
int type = typenum == NPY_UBYTE ? CV_8U : typenum == NPY_BYTE ? CV_8S :
typenum == NPY_USHORT ? CV_16U : typenum == NPY_SHORT ? CV_16S :
typenum == NPY_INT || typenum == NPY_LONG ? CV_32S :
typenum == NPY_FLOAT ? CV_32F :
typenum == NPY_DOUBLE ? CV_64F : -1;
if( type < 0 )
{
failmsg("toMat: Data type = %d is not supported", typenum);
}
int ndims = PyArray_NDIM(o);
if(ndims >= CV_MAX_DIM)
{
failmsg("toMat: Dimensionality (=%d) is too high", ndims);
}
int size[CV_MAX_DIM+1];
size_t step[CV_MAX_DIM+1], elemsize = CV_ELEM_SIZE1(type);
const npy_intp* _sizes = PyArray_DIMS(o);
const npy_intp* _strides = PyArray_STRIDES(o);
bool transposed = false;
for(int i = 0; i < ndims; i++)
{
size[i] = (int)_sizes[i];
step[i] = (size_t)_strides[i];
}
if( ndims == 0 || step[ndims-1] > elemsize ) {
size[ndims] = 1;
step[ndims] = elemsize;
ndims++;
}
if( ndims >= 2 && step[0] < step[1] )
{
std::swap(size[0], size[1]);
std::swap(step[0], step[1]);
transposed = true;
}
if( ndims == 3 && size[2] <= CV_CN_MAX && step[1] == elemsize*size[2] )
{
ndims--;
type |= CV_MAKETYPE(0, size[2]);
}
if( ndims > 2)
{
failmsg("toMat: Object has more than 2 dimensions");
}
m = Mat(ndims, size, type, PyArray_DATA(o), step);
if( m.data )
{
#if ( CV_MAJOR_VERSION < 3)
m.refcount = refcountFromPyObject(o);
m.addref(); // protect the original numpy array from deallocation
// (since Mat destructor will decrement the reference counter)
#else
m.u = g_numpyAllocator.allocate(o, ndims, size, type, step);
m.addref();
Py_INCREF(o);
//m.u->refcount = *refcountFromPyObject(o);
#endif
};
m.allocator = &g_numpyAllocator;
if( transposed )
{
Mat tmp;
tmp.allocator = &g_numpyAllocator;
transpose(m, tmp);
m = tmp;
}
return m;
}
PyObject* NDArrayConverter::toNDArray(const cv::Mat& m)
{
if( !m.data )
Py_RETURN_NONE;
Mat temp;
Mat *p = (Mat*)&m;
#if ( CV_MAJOR_VERSION < 3)
if(!p->refcount || p->allocator != &g_numpyAllocator)
{
temp.allocator = &g_numpyAllocator;
m.copyTo(temp);
p = &temp;
}
p->addref();
return pyObjectFromRefcount(p->refcount);
#else
if(!p->u || p->allocator != &g_numpyAllocator)
{
temp.allocator = &g_numpyAllocator;
m.copyTo(temp);
p = &temp;
}
//p->addref();
//return pyObjectFromRefcount(&p->u->refcount);
PyObject* o = (PyObject*) p->u->userdata;
Py_INCREF(o);
return o;
#endif
}
}
5. Compile source code and run
Compile DynaSLAM source code
cd DynaSLAM
chmod +x build.sh
./build.sh
If the latter two parameters are not given during operation, it is equivalent to running ORB-SLAM2.
If you only want to use the function of MaskRCNN but do not want to save the mask, then write no_save in PATH_MASK, otherwise give a folder address to save the Mask
./Examples/RGB-D/rgbd_tum Vocabulary/ORBvoc.txt Examples/RGB-D/TUM3.yaml /XXX/tum_dataset/ /XXX/tum_dataset/associations.txt masks/ output/
If it is found that Light Track has been unsuccessful and cannot be initialized after running, then increase the number of feature points in the ORB parameter setting, and generally change it to 3000 on github.
It's over, sprinkle flowers~