Panorama from multiple images

[akshayacharya97]

I want to create a panorama from a video by splitting it into frames and then stitch them using the conventional method of finding features.

This is my code for reference

import cv2
import numpy as np
import glob
import imutils


def draw_matches(img1, keypoints1, img2, keypoints2, matches):
    r, c = img1.shape[:2]
    r1, c1 = img2.shape[:2]

    # Create a blank image with the size of the first image + second image
    output_img = np.zeros((max([r, r1]), c + c1, 3), dtype='uint8')
    output_img[:r, :c, :] = np.dstack([img1])
    output_img[:r1, c:c + c1, :] = np.dstack([img2])

    # Go over all of the matching points and extract them
    for match in matches:
        img1_idx = match.queryIdx
        img2_idx = match.trainIdx
        (x1, y1) = keypoints1[img1_idx].pt
        (x2, y2) = keypoints2[img2_idx].pt

        # Draw circles on the keypoints
        cv2.circle(output_img, (int(x1), int(y1)), 4, (0, 255, 255), 1)
        cv2.circle(output_img, (int(x2) + c, int(y2)), 4, (0, 255, 255), 1)

        # Connect the same keypoints
        cv2.line(output_img, (int(x1), int(y1)), (int(x2) + c, int(y2)), (0, 255, 255), 1)

    return output_img


def warpImages(img1, img2, H):
    rows1, cols1 = img1.shape[:2]
    rows2, cols2 = img2.shape[:2]

    list_of_points_1 = np.float32([[0, 0], [0, rows1], [cols1, rows1], [cols1, 0]]).reshape(-1, 1, 2)
    temp_points = np.float32([[0, 0], [0, rows2], [cols2, rows2], [cols2, 0]]).reshape(-1, 1, 2)

    # When we have established a homography we need to warp perspective
    # Change field of view
    list_of_points_2 = cv2.perspectiveTransform(temp_points, H)

    list_of_points = np.concatenate((list_of_points_1, list_of_points_2), axis=0)

    [x_min, y_min] = np.int32(list_of_points.min(axis=0).ravel() - 0.5)
    [x_max, y_max] = np.int32(list_of_points.max(axis=0).ravel() + 0.5)

    translation_dist = [-x_min, -y_min]

    H_translation = np.array([[1, 0, translation_dist[0]], [0, 1, translation_dist[1]], [0, 0, 1]])

    output_img = cv2.warpPerspective(img2, H_translation.dot(H), (x_max - x_min, y_max - y_min))
    output_img[translation_dist[1]:rows1 + translation_dist[1], translation_dist[0]:cols1 + translation_dist[0]] = img1
    # print(output_img)

    return output_img


# Main program starts here


input_path = "/Users/akshayacharya/Desktop/Panorama/Bazinga/Test images for final/Highfps/*.jpg"
output_path = "Output/o4.jpg"
#input_path = "/Users/akshayacharya/Desktop/Panorama/Bazinga/Output/*.jpg"
#output_path = "Output/final.jpg"


input_img = glob.glob(input_path)
img_path = sorted(input_img)
print(img_path)
tmp = img_path[0]
flag = True

for i in range(1, len(img_path)):
    if flag:
        img1 = cv2.imread(tmp, cv2.COLOR_BGR2GRAY)
        img2 = cv2.imread(img_path[i], cv2.COLOR_BGR2GRAY)
        flag = False
    img1 = cv2.resize(img1, (1080, 720), fx=1, fy=1)
    img2 = cv2.imread(img_path[i], cv2.COLOR_BGR2GRAY)
    img2 = cv2.resize(img2, (1080, 720), fx=1, fy=1)

    orb = cv2.ORB_create(nfeatures=2000)

    keypoints1, descriptors1 = orb.detectAndCompute(img1, None)
    keypoints2, descriptors2 = orb.detectAndCompute(img2, None)

    # cv2.imshow('1',cv2.drawKeypoints(img1, keypoints1, None, (255, 0, 255)))
    # cv2.imshow('2',cv2.drawKeypoints(img2, keypoints2, None, (255,255, 255)))
    # cv2.waitKey(0)

    # Create a BFMatcher object.
    # It will find all of the matching keypoints on two images
    bf = cv2.BFMatcher_create(cv2.NORM_HAMMING)

    # Find matching points
    matches = bf.knnMatch(descriptors1, descriptors2, k=2)

    # print("Descriptor of the first keypoint: ")
    # print(descriptors1[0])
    # print(type(matches))

    all_matches = []
    for m, n in matches:
        all_matches.append(m)

    img3 = draw_matches(img1, keypoints1, img2, keypoints2, all_matches[:])
    # v2.imshow('Matches',img3)
    # cv2.waitKey(0)

    # Finding the best matches
    good = []
    for m, n in matches:
        if m.distance < 0.9 * n.distance:
            good.append(m)

    # cv2.imshow('Final1',cv2.drawKeypoints(img1, [keypoints1[m.queryIdx] for m in good], None, (255, 0, 255)))
    # cv2.imshow('Final2',cv2.drawKeypoints(img2, [keypoints2[m.queryIdx] for m in good], None, (255, 0, 255)))
    # cv2.waitKey(0)

    MIN_MATCH_COUNT = 10

    if len(good) > MIN_MATCH_COUNT:
        # Convert keypoints to an argument for findHomography
        src_pts = np.float32([keypoints1[m.queryIdx].pt for m in good]).reshape(-1, 1, 2)
        dst_pts = np.float32([keypoints2[m.trainIdx].pt for m in good]).reshape(-1, 1, 2)

        # Establish a homography
        M, _ = cv2.findHomography(src_pts, dst_pts, cv2.RANSAC, 5.0)

        result = warpImages(img2, img1, M)
        img1 = result
        print(f"Succesfully stitched until image{i + 1}")

#writeStatus = cv2.imwrite(output_path, result)
#if writeStatus is True:
#    print("image written")
#else:
#    print("problem")  # or raise exception, handle problem, etc.
#result = cv2.resize(result)
cv2.imshow("Hi", result)
cv2.waitKey(0)
#writeStatus = cv2.imwrite(output_path, result)



stitched = img1
stitched = cv2.copyMakeBorder(stitched, 10, 10, 10, 10,
                              cv2.BORDER_CONSTANT, (0, 0, 0))
# convert the stitched image to grayscale and threshold it
# such that all pixels greater than zero are set to 255
# (foreground) while all others remain 0 (background)
gray = cv2.cvtColor(stitched, cv2.COLOR_BGR2GRAY)
thresh = cv2.threshold(gray, 0, 255, cv2.THRESH_BINARY)[1]
# find all external contours in the threshold image then find
# the *largest* contour which will be the contour/outline of
# the stitched image
cnts = cv2.findContours(thresh.copy(), cv2.RETR_EXTERNAL,
                        cv2.CHAIN_APPROX_SIMPLE)
cnts = imutils.grab_contours(cnts)
c = max(cnts, key=cv2.contourArea)
# allocate memory for the mask which will contain the
# rectangular bounding box of the stitched image region
mask = np.zeros(thresh.shape, dtype="uint8")
(x, y, w, h) = cv2.boundingRect(c)
cv2.rectangle(mask, (x, y), (x + w, y + h), 255, -1)
# create two copies of the mask: one to serve as our actual
# minimum rectangular region and another to serve as a counter
# for how many pixels need to be removed to form the minimum
# rectangular region
minRect = mask.copy()
sub = mask.copy()
# keep looping until there are no non-zero pixels left in the
# subtracted image
while cv2.countNonZero(sub) > 0:
    # erode the minimum rectangular mask and then subtract
    # the thresholded image from the minimum rectangular mask
    # so we can count if there are any non-zero pixels left
    minRect = cv2.erode(minRect, None)
    sub = cv2.subtract(minRect, thresh)
# find contours in the minimum rectangular mask and then
# extract the bounding box (x, y)-coordinates
cnts = cv2.findContours(minRect.copy(), cv2.RETR_EXTERNAL,
                        cv2.CHAIN_APPROX_SIMPLE)
cnts = imutils.grab_contours(cnts)
c = max(cnts, key=cv2.contourArea)
(x, y, w, h) = cv2.boundingRect(c)
# use the bounding box coordinates to extract the our final
# stitched image
stitched = stitched[y:y + h, x:x + w]
#cv2.imwrite("cropped.jpg", stitched)
#writeStatus = cv2.imwrite(output_path, stitched)
#if writeStatus is True:
#    print("image written")
#else:
#    print("problem")  # or raise exception, handle problem, etc.
stitched = cv2.resize(stitched, (2000,1500))
cv2.imshow("cropped", stitched)
cv2.waitKey(0)

However, its not giving me the right output. I have attached the image for reference. Can someone guide me as to how I could get the right panorama? The images for the source are obtained by splitting a video into frames and then using these as input images.