How to make pagination work in realtime?

I tried to make pagination work in realtime by creating a FirestoreBoundaryCallback:-

class FirestoreBoundaryCallback<T>(
        private val baseQuery: Query,
        private val factory: FirestoreQueryDataSource.Factory,
        private val lifecycleOwner: LifecycleOwner): PagedList.BoundaryCallback<QueryItemOrException<T>>() {

    private val allLiveData = mutableListOf<FirebaseQueryLiveData>()
    private val mutableLoadingState = MutableLiveData<LoadingState>()

    val loadingState: LiveData<LoadingState>
        get() = mutableLoadingState

    override fun onZeroItemsLoaded() {
        allLiveData.clear()
        mutableLoadingState.value = LoadingState.LOADING_INITIAL
        val query = baseQuery.limit(50)
        val liveData = FirebaseQueryLiveData(query)
        liveData.observe(lifecycleOwner, Observer {
            mergeAllDocs()
            if (mutableLoadingState.value != LoadingState.LOADED) {
                mutableLoadingState.value = LoadingState.LOADED
            }
        })
        allLiveData.add(liveData)
    }

    override fun onItemAtEndLoaded(itemAtEnd: QueryItemOrException<T>) {
        if (allLiveData.isNotEmpty() && allLiveData.last().value?.data?.documents?.isNotEmpty() == true) {
            val lastDocument = allLiveData.last().value?.data?.documents?.last()
            if (lastDocument != null) {
                val query = baseQuery.startAfter(lastDocument).limit(50)
                val liveData = FirebaseQueryLiveData(query)
                mutableLoadingState.value = LoadingState.LOADING_MORE
                liveData.observe(lifecycleOwner, Observer {
                    mergeAllDocs()
                    if (mutableLoadingState.value != LoadingState.LOADED) {
                        mutableLoadingState.value = LoadingState.LOADED
                    }
                })
                allLiveData.add(liveData)
            }
        }
    }

    fun mergeAllDocs() {
        val items = mutableListOf<DocumentSnapshot>()
        allLiveData.forEach{
            val docs = it.value?.data?.documents
            if (docs != null) {
                items.addAll(docs)
            }
        }
        factory.setItems(items)
    }

    override fun onItemAtFrontLoaded(itemAtFront: QueryItemOrException<T>) {
    }
}

And then I modified the FirestoreQueryDataSource in the following way:-

class FirestoreQueryDataSource private constructor(
    private val documentSnapshots: List<DocumentSnapshot>
) : PageKeyedDataSource<PageKey, DocumentSnapshot>() {

    companion object {
        private const val TAG = "FirestoreQueryDataSrc"
    }

    class Factory(private val query: Query, private val source: Source) : DataSource.Factory<PageKey, DocumentSnapshot>() {
        val sourceLiveData = MutableLiveData<FirestoreQueryDataSource>()
        var documentSnapshots: List<DocumentSnapshot> = mutableListOf()

        fun setItems(items: List<DocumentSnapshot>) {
            sourceLiveData.value?.invalidate()
            documentSnapshots = items
            sourceLiveData.postValue(FirestoreQueryDataSource(documentSnapshots))
        }

        override fun create(): DataSource<PageKey, DocumentSnapshot> {
            val dataSource = FirestoreQueryDataSource(documentSnapshots)
            sourceLiveData.postValue(dataSource)
            return dataSource
        }
    }

    override fun loadInitial(
            params: LoadInitialParams<PageKey>,
            callback: LoadInitialCallback<PageKey, DocumentSnapshot>) {

        val firstPageDocSnapshots = documentSnapshots.take(params.requestedLoadSize)
        val nextPageKey = getNextPageKey(firstPageDocSnapshots)
        callback.onResult(firstPageDocSnapshots, null, nextPageKey)
    }

    override fun loadAfter(
            params: LoadParams<PageKey>,
            callback: LoadCallback<PageKey, DocumentSnapshot>) {

        val startAfterIndex = documentSnapshots.indexOf(params.key.startAfterDoc)
        var endIndex = startAfterIndex + params.requestedLoadSize
        if (endIndex > documentSnapshots.size) {
            endIndex = documentSnapshots.size - 1;
        }
        val afterInitialPageDocs = documentSnapshots.subList(startAfterIndex, endIndex)
        val nextPageKey = getNextPageKey(afterInitialPageDocs)
        callback.onResult(afterInitialPageDocs, nextPageKey)
    }

    override fun loadBefore(
            params: LoadParams<PageKey>,
            callback: LoadCallback<PageKey, DocumentSnapshot>) {
        // The paging here only understands how to append new items to the
        // results, not prepend items from earlier pages.
        callback.onResult(emptyList(), null)
    }

    private fun getNextPageKey(documents: List<DocumentSnapshot>): PageKey? {
        return if (documents.isNotEmpty()) {
            PageKey(documents.last())
        } else {
            null
        }
    }

}

data class PageKey(val startAfterDoc: DocumentSnapshot)

In my ViewModel, this is what I return:-

 val sourceFactory = FirestoreQueryDataSource.Factory(query, Source.DEFAULT)
        val deserializedDataSourceFactory = sourceFactory.map { snapshot ->
            try {
                val item = QueryItem(Deserializer.deserialize(snapshot, Record::class.java), snapshot.id)
                item.item.id = snapshot.id
                QueryItemOrException(item, null)
            } catch (e: Exception) {
                Log.e(TAG, "Error while deserializing order", e)
                QueryItemOrException<PosOrder>(null, e)
            }
        }
        val boundaryCallback = FirestoreBoundaryCallback<Record>(query, sourceFactory, lifecycleOwner)
        val livePagedList = LivePagedListBuilder(deserializedDataSourceFactory, 30)
                .setFetchExecutor(executors.cpuExecutorService)
                .setBoundaryCallback(boundaryCallback)
                .build()
        return Listing(
                pagedList = livePagedList,
                loadingState = boundaryCallback.loadingState,
                refresh = {
                    sourceFactory.sourceLiveData.value?.invalidate()
                }
        )

It works but with really bad performance. What am I doing wrong? Also I am concerned that when a new record is inserted, the first page will lose its last record (since the limit is fixed for the first query) and second page will continue to start AFTER the last record of the older first page.