xref: /aosp_15_r20/external/libvpx/tools/3D-Reconstruction/MotionEST/SearchSmooth.py (revision fb1b10ab9aebc7c7068eedab379b749d7e3900be)

##  Copyright (c) 2020 The WebM project authors. All Rights Reserved.
##
##  Use of this source code is governed by a BSD-style license
##  that can be found in the LICENSE file in the root of the source
##  tree. An additional intellectual property rights grant can be found
##  in the file PATENTS.  All contributing project authors may
##  be found in the AUTHORS file in the root of the source tree.
##

# coding: utf-8
import numpy as np
import numpy.linalg as LA
from Util import MSE
from MotionEST import MotionEST
"""Search & Smooth Model with Adapt Weights"""


class SearchSmoothAdapt(MotionEST):
  """
    Constructor:
        cur_f: current frame
        ref_f: reference frame
        blk_sz: block size
        wnd_size: search window size
        beta: neigbor loss weight
        max_iter: maximum number of iterations
        metric: metric to compare the blocks distrotion
    """

  def __init__(self, cur_f, ref_f, blk_size, search, max_iter=100):
    self.search = search
    self.max_iter = max_iter
    super(SearchSmoothAdapt, self).__init__(cur_f, ref_f, blk_size)

  """
    get local diffiencial of refernce
    """

  def getRefLocalDiff(self, mvs):
    m, n = self.num_row, self.num_col
    localDiff = [[] for _ in xrange(m)]
    blk_sz = self.blk_sz
    for r in xrange(m):
      for c in xrange(n):
        I_row = 0
        I_col = 0
        #get ssd surface
        count = 0
        center = self.cur_yuv[r * blk_sz:(r + 1) * blk_sz,
                              c * blk_sz:(c + 1) * blk_sz, 0]
        ty = np.clip(r * blk_sz + int(mvs[r, c, 0]), 0, self.height - blk_sz)
        tx = np.clip(c * blk_sz + int(mvs[r, c, 1]), 0, self.width - blk_sz)
        target = self.ref_yuv[ty:ty + blk_sz, tx:tx + blk_sz, 0]
        for y, x in {(ty - blk_sz, tx), (ty + blk_sz, tx)}:
          if 0 <= y < self.height - blk_sz and 0 <= x < self.width - blk_sz:
            nb = self.ref_yuv[y:y + blk_sz, x:x + blk_sz, 0]
            I_row += np.sum(np.abs(nb - center)) - np.sum(
                np.abs(target - center))
            count += 1
        I_row //= (count * blk_sz * blk_sz)
        count = 0
        for y, x in {(ty, tx - blk_sz), (ty, tx + blk_sz)}:
          if 0 <= y < self.height - blk_sz and 0 <= x < self.width - blk_sz:
            nb = self.ref_yuv[y:y + blk_sz, x:x + blk_sz, 0]
            I_col += np.sum(np.abs(nb - center)) - np.sum(
                np.abs(target - center))
            count += 1
        I_col //= (count * blk_sz * blk_sz)
        localDiff[r].append(
            np.array([[I_row * I_row, I_row * I_col],
                      [I_col * I_row, I_col * I_col]]))
    return localDiff

  """
    add smooth constraint
    """

  def smooth(self, uvs, mvs):
    sm_uvs = np.zeros(uvs.shape)
    blk_sz = self.blk_sz
    for r in xrange(self.num_row):
      for c in xrange(self.num_col):
        nb_uv = np.array([0.0, 0.0])
        for i, j in {(r - 1, c), (r + 1, c), (r, c - 1), (r, c + 1)}:
          if 0 <= i < self.num_row and 0 <= j < self.num_col:
            nb_uv += uvs[i, j] / 6.0
          else:
            nb_uv += uvs[r, c] / 6.0
        for i, j in {(r - 1, c - 1), (r - 1, c + 1), (r + 1, c - 1),
                     (r + 1, c + 1)}:
          if 0 <= i < self.num_row and 0 <= j < self.num_col:
            nb_uv += uvs[i, j] / 12.0
          else:
            nb_uv += uvs[r, c] / 12.0
        ssd_nb = self.block_dist(r, c, self.blk_sz * nb_uv)
        mv = mvs[r, c]
        ssd_mv = self.block_dist(r, c, mv)
        alpha = (ssd_nb - ssd_mv) / (ssd_mv + 1e-6)
        M = alpha * self.localDiff[r][c]
        P = M + np.identity(2)
        inv_P = LA.inv(P)
        sm_uvs[r, c] = np.dot(inv_P, nb_uv) + np.dot(
            np.matmul(inv_P, M), mv / blk_sz)
    return sm_uvs

  def block_matching(self):
    self.search.motion_field_estimation()

  def motion_field_estimation(self):
    self.localDiff = self.getRefLocalDiff(self.search.mf)
    #get matching results
    mvs = self.search.mf
    #add smoothness constraint
    uvs = mvs / self.blk_sz
    for _ in xrange(self.max_iter):
      uvs = self.smooth(uvs, mvs)
    self.mf = uvs * self.blk_sz


"""Search & Smooth Model with Fixed Weights"""


class SearchSmoothFix(MotionEST):
  """
    Constructor:
        cur_f: current frame
        ref_f: reference frame
        blk_sz: block size
        wnd_size: search window size
        beta: neigbor loss weight
        max_iter: maximum number of iterations
        metric: metric to compare the blocks distrotion
    """

  def __init__(self, cur_f, ref_f, blk_size, search, beta, max_iter=100):
    self.search = search
    self.max_iter = max_iter
    self.beta = beta
    super(SearchSmoothFix, self).__init__(cur_f, ref_f, blk_size)

  """
    get local diffiencial of refernce
    """

  def getRefLocalDiff(self, mvs):
    m, n = self.num_row, self.num_col
    localDiff = [[] for _ in xrange(m)]
    blk_sz = self.blk_sz
    for r in xrange(m):
      for c in xrange(n):
        I_row = 0
        I_col = 0
        #get ssd surface
        count = 0
        center = self.cur_yuv[r * blk_sz:(r + 1) * blk_sz,
                              c * blk_sz:(c + 1) * blk_sz, 0]
        ty = np.clip(r * blk_sz + int(mvs[r, c, 0]), 0, self.height - blk_sz)
        tx = np.clip(c * blk_sz + int(mvs[r, c, 1]), 0, self.width - blk_sz)
        target = self.ref_yuv[ty:ty + blk_sz, tx:tx + blk_sz, 0]
        for y, x in {(ty - blk_sz, tx), (ty + blk_sz, tx)}:
          if 0 <= y < self.height - blk_sz and 0 <= x < self.width - blk_sz:
            nb = self.ref_yuv[y:y + blk_sz, x:x + blk_sz, 0]
            I_row += np.sum(np.abs(nb - center)) - np.sum(
                np.abs(target - center))
            count += 1
        I_row //= (count * blk_sz * blk_sz)
        count = 0
        for y, x in {(ty, tx - blk_sz), (ty, tx + blk_sz)}:
          if 0 <= y < self.height - blk_sz and 0 <= x < self.width - blk_sz:
            nb = self.ref_yuv[y:y + blk_sz, x:x + blk_sz, 0]
            I_col += np.sum(np.abs(nb - center)) - np.sum(
                np.abs(target - center))
            count += 1
        I_col //= (count * blk_sz * blk_sz)
        localDiff[r].append(
            np.array([[I_row * I_row, I_row * I_col],
                      [I_col * I_row, I_col * I_col]]))
    return localDiff

  """
    add smooth constraint
    """

  def smooth(self, uvs, mvs):
    sm_uvs = np.zeros(uvs.shape)
    blk_sz = self.blk_sz
    for r in xrange(self.num_row):
      for c in xrange(self.num_col):
        nb_uv = np.array([0.0, 0.0])
        for i, j in {(r - 1, c), (r + 1, c), (r, c - 1), (r, c + 1)}:
          if 0 <= i < self.num_row and 0 <= j < self.num_col:
            nb_uv += uvs[i, j] / 6.0
          else:
            nb_uv += uvs[r, c] / 6.0
        for i, j in {(r - 1, c - 1), (r - 1, c + 1), (r + 1, c - 1),
                     (r + 1, c + 1)}:
          if 0 <= i < self.num_row and 0 <= j < self.num_col:
            nb_uv += uvs[i, j] / 12.0
          else:
            nb_uv += uvs[r, c] / 12.0
        mv = mvs[r, c] / blk_sz
        M = self.localDiff[r][c]
        P = M + self.beta * np.identity(2)
        inv_P = LA.inv(P)
        sm_uvs[r, c] = np.dot(inv_P, self.beta * nb_uv) + np.dot(
            np.matmul(inv_P, M), mv)
    return sm_uvs

  def block_matching(self):
    self.search.motion_field_estimation()

  def motion_field_estimation(self):
    #get local structure
    self.localDiff = self.getRefLocalDiff(self.search.mf)
    #get matching results
    mvs = self.search.mf
    #add smoothness constraint
    uvs = mvs / self.blk_sz
    for _ in xrange(self.max_iter):
      uvs = self.smooth(uvs, mvs)
    self.mf = uvs * self.blk_sz