android - How to have similar mechanism of center-crop on ExoPlayer's PlayerView , but not on the center?

Question

Background

We record a video of the user's face, and usually the face is located at the upper half of the video.

Later we wish to view the video, but the aspect ratio of the PlayerView might be different than the one of the video, so there needs to be some scaling and cropping.

The problem

The only way I've found to scale the PlayerView so that it will be shown in the entire space it has but keeping the aspect ratio (which will result in cropping when needed, of course) , is by using app:resize_mode="zoom" . Here's a sample of how it works with center-crop: http://s000.tinyupload.com/?file_id=00574047057406286563 . The more the Views that show the content have a similar aspect ratio, the less cropping is needed.

But this is only for the center, meaning it takes a point of 0.5x0.5 of the video, and scale-crops from that point. This causes many cases of losing the important content of the video.

For example, if we have a video that was taken in portrait, and we have a square PlayerView and want to show the top area, this is the part that will be visible:

Of course, if the content itself is square, and the views are also square, it should show the entire content, without cropping.

What I've tried

I've tried searching over the Internet, StackOverflow (here) and on Github, but I couldn't find how to do it. The only clue I've found is about AspectRatioFrameLayout and AspectRatioTextureView, but I didn't find how to use them for this task, if it's even possible.

I was told (here) that I should use a normal TextureView , and provide it directly to SimpleExoPlayer using SimpleExoPlayer.setVideoTextureView. And to set a special transformation to it using TextureView.setTransform.

After a lot of trying what is best to use (and looking at video-crop repository , SuperImageView repository , and JCropImageView repository which have examples of scale/crop of ImageView and video), I've published a working sample that seems to show the video correctly, but I'm still not sure about it, as I also use an ImageView that's shown on top of it before it starts playing (to have a nicer transition instead of black content).

Here's the current code:

class MainActivity : AppCompatActivity() {
    private val imageResId = R.drawable.test
    private val videoResId = R.raw.test
    private val percentageY = 0.2f
    private var player: SimpleExoPlayer? = null


    override fun onCreate(savedInstanceState: Bundle?) {
        window.setBackgroundDrawable(ColorDrawable(0xff000000.toInt()))
        super.onCreate(savedInstanceState)
        if (cache == null) {
            cache = SimpleCache(File(cacheDir, "media"), LeastRecentlyUsedCacheEvictor(MAX_PREVIEW_CACHE_SIZE_IN_BYTES))
        }
        setContentView(R.layout.activity_main)
//        imageView.visibility = View.INVISIBLE
        imageView.setImageResource(imageResId)
        imageView.doOnPreDraw {
            imageView.imageMatrix = prepareMatrixForImageView(imageView, imageView.drawable.intrinsicWidth.toFloat(), imageView.drawable.intrinsicHeight.toFloat())
//            imageView.imageMatrix = prepareMatrix(imageView, imageView.drawable.intrinsicWidth.toFloat(), imageView.drawable.intrinsicHeight.toFloat())
//            imageView.visibility = View.VISIBLE
        }
    }

    override fun onStart() {
        super.onStart()
        playVideo()
    }

    private fun prepareMatrix(view: View, contentWidth: Float, contentHeight: Float): Matrix {
        var scaleX = 1.0f
        var scaleY = 1.0f
        val viewWidth = view.measuredWidth.toFloat()
        val viewHeight = view.measuredHeight.toFloat()
        Log.d("AppLog", "viewWidth $viewWidth viewHeight $viewHeight contentWidth:$contentWidth contentHeight:$contentHeight")
        if (contentWidth > viewWidth && contentHeight > viewHeight) {
            scaleX = contentWidth / viewWidth
            scaleY = contentHeight / viewHeight
        } else if (contentWidth < viewWidth && contentHeight < viewHeight) {
            scaleY = viewWidth / contentWidth
            scaleX = viewHeight / contentHeight
        } else if (viewWidth > contentWidth)
            scaleY = viewWidth / contentWidth / (viewHeight / contentHeight)
        else if (viewHeight > contentHeight)
            scaleX = viewHeight / contentHeight / (viewWidth / contentWidth)
        val matrix = Matrix()
        val pivotPercentageX = 0.5f
        val pivotPercentageY = percentageY

        matrix.setScale(scaleX, scaleY, viewWidth * pivotPercentageX, viewHeight * pivotPercentageY)
        return matrix
    }

    private fun prepareMatrixForVideo(view: View, contentWidth: Float, contentHeight: Float): Matrix {
        val msWidth = view.measuredWidth
        val msHeight = view.measuredHeight
        val matrix = Matrix()
        matrix.setScale(1f, (contentHeight / contentWidth) * (msWidth.toFloat() / msHeight), msWidth / 2f, percentageY * msHeight) /*,msWidth/2f,msHeight/2f*/
        return matrix
    }

    private fun prepareMatrixForImageView(view: View, contentWidth: Float, contentHeight: Float): Matrix {
        val dw = contentWidth
        val dh = contentHeight
        val msWidth = view.measuredWidth
        val msHeight = view.measuredHeight
//        Log.d("AppLog", "viewWidth $msWidth viewHeight $msHeight contentWidth:$contentWidth contentHeight:$contentHeight")
        val scalew = msWidth.toFloat() / dw
        val theoryh = (dh * scalew).toInt()
        val scaleh = msHeight.toFloat() / dh
        val theoryw = (dw * scaleh).toInt()
        val scale: Float
        var dx = 0
        var dy = 0
        if (scalew > scaleh) { // fit width
            scale = scalew
//            dy = ((msHeight - theoryh) * 0.0f + 0.5f).toInt() // + 0.5f for rounding
        } else {
            scale = scaleh
            dx = ((msWidth - theoryw) * 0.5f + 0.5f).toInt() // + 0.5f for rounding
        }
        dy = ((msHeight - theoryh) * percentageY + 0.5f).toInt() // + 0.5f for rounding
        val matrix = Matrix()
//        Log.d("AppLog", "scale:$scale dx:$dx dy:$dy")
        matrix.setScale(scale, scale)
        matrix.postTranslate(dx.toFloat(), dy.toFloat())
        return matrix
    }

    private fun playVideo() {
        player = ExoPlayerFactory.newSimpleInstance(this@MainActivity, DefaultTrackSelector())
        player!!.setVideoTextureView(textureView)
        player!!.addVideoListener(object : VideoListener {
            override fun onVideoSizeChanged(width: Int, height: Int, unappliedRotationDegrees: Int, pixelWidthHeightRatio: Float) {
                super.onVideoSizeChanged(width, height, unappliedRotationDegrees, pixelWidthHeightRatio)
                Log.d("AppLog", "onVideoSizeChanged: $width $height")
                val videoWidth = if (unappliedRotationDegrees % 180 == 0) width else height
                val videoHeight = if (unappliedRotationDegrees % 180 == 0) height else width
                val matrix = prepareMatrixForVideo(textureView, videoWidth.toFloat(), videoHeight.toFloat())
                textureView.setTransform(matrix)
            }

            override fun onRenderedFirstFrame() {
                Log.d("AppLog", "onRenderedFirstFrame")
                player!!.removeVideoListener(this)
//                imageView.animate().alpha(0f).setDuration(5000).start()
                imageView.visibility = View.INVISIBLE
            }
        })
        player!!.volume = 0f
        player!!.repeatMode = Player.REPEAT_MODE_ALL
        player!!.playRawVideo(this, videoResId)
        player!!.playWhenReady = true
        //        player!!.playVideoFromUrl(this, "https://sample-videos.com/video123/mkv/240/big_buck_bunny_240p_20mb.mkv", cache!!)
        //        player!!.playVideoFromUrl(this, "https://sample-videos.com/video123/mkv/720/big_buck_bunny_720p_1mb.mkv", cache!!)
        //        player!!.playVideoFromUrl(this@MainActivity, "https://sample-videos.com/video123/mkv/720/big_buck_bunny_720p_1mb.mkv")
    }

    override fun onStop() {
        super.onStop()
        player!!.setVideoTextureView(null)
        //        playerView.player = null
        player!!.release()
        player = null
    }

    companion object {
        const val MAX_PREVIEW_CACHE_SIZE_IN_BYTES = 20L * 1024L * 1024L
        var cache: com.google.android.exoplayer2.upstream.cache.Cache? = null

        @JvmStatic
        fun getUserAgent(context: Context): String {
            val packageManager = context.packageManager
            val info = packageManager.getPackageInfo(context.packageName, 0)
            val appName = info.applicationInfo.loadLabel(packageManager).toString()
            return Util.getUserAgent(context, appName)
        }
    }

    fun SimpleExoPlayer.playRawVideo(context: Context, @RawRes rawVideoRes: Int) {
        val dataSpec = DataSpec(RawResourceDataSource.buildRawResourceUri(rawVideoRes))
        val rawResourceDataSource = RawResourceDataSource(context)
        rawResourceDataSource.open(dataSpec)
        val factory: DataSource.Factory = DataSource.Factory { rawResourceDataSource }
        prepare(LoopingMediaSource(ExtractorMediaSource.Factory(factory).createMediaSource(rawResourceDataSource.uri)))
    }

    fun SimpleExoPlayer.playVideoFromUrl(context: Context, url: String, cache: Cache? = null) = playVideoFromUri(context, Uri.parse(url), cache)

    fun SimpleExoPlayer.playVideoFile(context: Context, file: File) = playVideoFromUri(context, Uri.fromFile(file))

    fun SimpleExoPlayer.playVideoFromUri(context: Context, uri: Uri, cache: Cache? = null) {
        val factory = if (cache != null)
            CacheDataSourceFactory(cache, DefaultHttpDataSourceFactory(getUserAgent(context)))
        else
            DefaultDataSourceFactory(context, MainActivity.getUserAgent(context))
        val mediaSource = ExtractorMediaSource.Factory(factory).createMediaSource(uri)
        prepare(mediaSource)
    }
}

I had various issues on trying this till I got to the current situation, and I've updated this question multiple times accordingly. Now it even works with the percentageY I talked about, so I could set it to be from 20% of the top of the video, if I wish. However, I still think that it has a big chance that something is wrong, because when I tried to set it to 50% , I've noticed that the content might not fit the entire View.

I even looked at the source code of ImageView (here), to see how center-crop is used. When applied to the ImageView, it still worked as center-crop, but when I used the same technique on the video, it gave me a very wrong result.

The questions

My goal here was to show both ImageVie

Answer 1

The question is how to manipulate an image like ImageView.ScaleType.CENTER_CROP but to shift the focus from the center to another location that is 20% from the top of the image. First, let's look at what CENTER_CROP does:

From the documentation:

CENTER_CROP

Scale the image uniformly (maintain the image's aspect ratio) so that both dimensions (width and height) of the image will be equal to or larger than the corresponding dimension of the view (minus padding). The image is then centered in the view. From XML, use this syntax: android:scaleType="centerCrop".

In other words, scale the image without distortion such that either the width or height of the image (or both width and height) fit within the view so that the view is completely filled with the image (no gaps.)

Another way to think of this is that the center of the image is "pinned" to the center of the view. The image is then scaled to meet the criteria above.

In the following video, the white lines mark the center of the image; the red lines mark the center of the view. The scale type is CENTER_CROP. Notice how the center points of the image and the view coincide. As the view changes size, these two points continue to overlap and always appear at the center of the view regardless of the view size.

So, what does it mean to have center crop-like behavior at a different location such as 20% from the top? Like center crop, we can specify that the point that is 20% from the top of the image and the point that 20% from the top of the view will be "pinned" like the 50% point is "pinned" in center crop. The horizontal location of this point remains at 50% of the image and view. The image can now be scaled to satisfy the other conditions of center crop which specify that either the width and/or height of the image will fit the view with no gaps. (Size of view is understood to be the view size less padding.)

Here is a short video of this 20% crop behavior. In this video, the white lines show the middle of the image, the red lines show the pinned point in the view and the blue line that shows behind the horizontal red line identifies 20% from the top of the image. (Demo project is on GitHub.

Here is the result showing the full image that was supplied and the video in a square frame that transition from the still image. .

MainActivity.kt
prepareMatrix() is the method that does the work to determine how to scale/crop the image. There is some additional work to be done with the video since it appears that the video is made to fit the TextureViewas a scale type "FIT_XY" when it is assigned to the TextureView. Because of this scaling, the media size must be restored before prepareMatrix() is called for the video

class MainActivity : AppCompatActivity() {
    private val imageResId = R.drawable.test
    private val videoResId = R.raw.test
    private var player: SimpleExoPlayer? = null
    private val mFocalPoint = PointF(0.5f, 0.2f)

    override fun onCreate(savedInstanceState: Bundle?) {
        window.setBackgroundDrawable(ColorDrawable(0xff000000.toInt()))
        super.onCreate(savedInstanceState)
        if (cache == null) {
            cache = SimpleCache(File(cacheDir, "media"), LeastRecentlyUsedCacheEvictor(MAX_PREVIEW_CACHE_SIZE_IN_BYTES))
        }
        setContentView(R.layout.activity_main)
        //        imageView.visibility = View.INVISIBLE
        imageView.setImageResource(imageResId)
        imageView.doOnPreDraw {
            imageView.scaleType = ImageView.ScaleType.MATRIX
            val imageWidth: Float = ContextCompat.getDrawable(this, imageResId)!!.intrinsicWidth.toFloat()
            val imageHeight: Float = ContextCompat.getDrawable(this, imageResId)!!.intrinsicHeight.toFloat()
            imageView.imageMatrix = prepareMatrix(imageView, imageWidth, imageHeight, mFocalPoint, Matrix())
            val b = BitmapFactory.decodeResource(resources, imageResId)
            val d = BitmapDrawable(resources, b.copy(Bitmap.Config.ARGB_8888, true))
            val c = Canvas(d.bitmap)
            val p = Paint()
            p.color = resources.getColor(android.R.color.holo_red_dark)
            p.style = Paint.Style.STROKE
            val strokeWidth = 10
            p.strokeWidth = strokeWidth.toFloat()
            // Horizontal line
            c.drawLine(0f, imageHeight * mFocalPoint.y, imageWidth, imageHeight * mFocalPoint.y, p)
            // Vertical line
            c.drawLine(imageWidth * mFocalPoint.x, 0f, imageWidth * mFocalPoint.x, imageHeight, p)
            // Line in horizontal and vertical center
            p.color = resources.getColor(android.R.color.white)
            c.drawLine(imageWidth / 2, 0f, imageWidth / 2, imageHeight, p)
            c.drawLine(0f, imageHeight / 2, imageWidth, imageHeight / 2, p)

            imageView.setImageBitmap(d.bitmap)
            imageViewFull.setImageBitmap(d.bitmap)
        }
    }

    fun startPlay(view: View) {
        playVideo()
    }

    private fun getViewWidth(view: View): Float {
        return (view.width - view.paddingStart - view.paddingEnd).toFloat()
    }

    private fun getViewHeight(view: View): Float {
        return (view.height - view.paddingTop - view.paddingBottom).toFloat()
    }

    private fun prepareMatrix(targetView: View, mediaWidth: Float, mediaHeight: Float,
                              focalPoint: PointF, matrix: Matrix): Matrix {
        if (targetView.visibility != View.VISIBLE) {
            return matrix
        }
        val viewHeight = getViewHeight(targetView)
        val viewWidth = getViewWidth(targetView)
        val scaleFactorY = viewHeight / mediaHeight
        val scaleFactor: Float
        val px: Float
        val py: Float
        if (mediaWidth * scaleFactorY >= viewWidth) {
            // Fit height
            scaleFactor = scaleFactorY
            px = -(mediaWidth * scaleFactor - viewWidth) * focalPoint.x / (1 - scaleFactor)
            py = 0f
        } else {
            // Fit width
            scaleFactor = viewWidth / mediaWidth
            px = 0f
            py = -(mediaHeight * scaleFactor - viewHeight) * focalPoint.y / (1 - scaleFactor)
        }
        matrix.postScale(scaleFactor, scaleFactor, px, py)
        return matrix
    }

    private fun playVideo() {
        player = ExoPlayerFactory.newSimpleInstance(this@MainActivity, DefaultTrackSelector())
        player!!.setVideoTextureView(textureView)
        player!!.addVideoListener(object : VideoListener {
            override fun onVideoSizeChanged(width: Int, height: Int, unappliedRotationDegrees: Int, pixelWidthHeightRatio: Float) {
                super.onVideoSizeChanged(width, height, unappliedRotationDegrees, pixelWidthHeightRatio)
                val matrix = Matrix()
                // Restore true media size for further manipulation.
                matrix.setScale(width / getViewWidth(textureView), height / getViewHeight(textureView))
                textureView.setTransform(prepareMatrix(textureView, width.toFloat(), height.toFloat(), mFocalPoint, matrix))
            }

            override fun onRenderedFirstFrame() {
                Log.d("AppLog", "onRenderedFirstFrame")
                player!!.removeVideoListener(this)
                imageView.animate().alpha(0f).setDuration(2000).start()
                imageView.visibility = View.INVISIBLE
            }
        })
        player!!.volume = 0f
        player!!.repeatMode = Player.REPEAT_MODE_ALL
        player!!.playRawVideo(this, videoResId)
        player!!.playWhenReady = true
        //        player!!.playVideoFromUrl(this, "https://sample-videos.com/video123/mkv/240/big_buck_bunny_240p_20mb.mkv", cache!!)
        //        player!!.playVideoFromUrl(this, "https://sample-videos.com/video123/mkv/720/big_buck_bunny_720p_1mb.mkv", cache!!)
        //        player!!.playVideoFromUrl(this@MainActivity, "https://sample-videos.com/video123/mkv/720/big_buck_bunny_720p_1mb.mkv")
    }

    override fun onStop() {
        super.onStop()
        if (player != null) {
            player!!.setVideoTextureView(null)
            //        playerView.player = null
            player!!.release()
            player = null
        }
    }

    companion object {
        const val MAX_PREVIEW_CACHE_SIZE_IN_BYTES = 20L * 1024L * 1024L
        var cache: com.google.android.exoplayer2.upstream.cache.Cache? = null

        @JvmStatic
        fun getUserAgent(context: Context): String {
            val packageManager = context.packageManager
            val info = packageManager.getPackageInfo(context.packageName, 0)
            val appName = info.applicationInfo.loadLabel(packageManager).toString()
            return Util.getUserAgent(context, appName)
        }
    }

    fun SimpleExoPlayer.playRawVideo(context: Context, @RawRes rawVideoRes: Int) {
        val dataSpec = DataSpec(RawResourceDataSource.buildRawResourceUri(rawVideoRes))
        val rawResourceDataSource = RawResourceDataSource(context)
        rawResourceDataSource.open(dataSpec)
        val factory: DataSource.Factory = DataSource.Factory { rawResourceDataSource }
        prepare(LoopingMediaSource(ExtractorMediaSource.Factory(factory).createMediaSource(rawResourceDataSource.uri)))
    }

    fun SimpleExoPlayer.playVideoFromUrl(context: Context, url: String, cache: Cache? = null) = playVideoFromUri(context, Uri.parse(url), cache)

    fun SimpleExoPlayer.playVideoFile(context: Context, file: File) = playVideoFromUri(context, Uri.fromFile(file))

    fun SimpleExoPlayer.playVideoFromUri(context: Context, uri: Uri, cache: Cache? = null) {
        val factory = if (cache != null)
            CacheDataSourceFactory(cache, DefaultHttpDataSourceFactory(getUserAgent(context)))
        else
            DefaultDataSourceFactory(context, MainActivity.getUserAgent(context))
        val mediaSource = ExtractorMediaSource.Factory(factory).createMediaSource(uri)
        prepare(mediaSource)
    }
}