void
exaGlyphsInit(ScreenPtr pScreen)
{
    ExaScreenPriv(pScreen);
    int i = 0;

    memset(pExaScr->glyphCaches, 0, sizeof(pExaScr->glyphCaches));

    pExaScr->glyphCaches[i].format = PICT_a8;
    pExaScr->glyphCaches[i].glyphWidth = pExaScr->glyphCaches[i].glyphHeight =
        16;
    i++;
    pExaScr->glyphCaches[i].format = PICT_a8;
    pExaScr->glyphCaches[i].glyphWidth = pExaScr->glyphCaches[i].glyphHeight =
        32;
    i++;
    pExaScr->glyphCaches[i].format = PICT_a8r8g8b8;
    pExaScr->glyphCaches[i].glyphWidth = pExaScr->glyphCaches[i].glyphHeight =
        16;
    i++;
    pExaScr->glyphCaches[i].format = PICT_a8r8g8b8;
    pExaScr->glyphCaches[i].glyphWidth = pExaScr->glyphCaches[i].glyphHeight =
        32;
    i++;

    assert(i == EXA_NUM_GLYPH_CACHES);

    for (i = 0; i < EXA_NUM_GLYPH_CACHES; i++) {
        pExaScr->glyphCaches[i].columns =
            CACHE_PICTURE_WIDTH / pExaScr->glyphCaches[i].glyphWidth;
        pExaScr->glyphCaches[i].size = 256;
        pExaScr->glyphCaches[i].hashSize = 557;
    }
}

/**
 * Ejects all offscreen areas, and uninitializes the offscreen memory manager.
 */
void
ExaOffscreenSwapOut (ScreenPtr pScreen)
{
    ExaScreenPriv (pScreen);

    ExaOffscreenValidate (pScreen);
    /* loop until a single free area spans the space */
    for (;;)
    {
	ExaOffscreenArea *area = pExaScr->info->offScreenAreas;

	if (!area)
	    break;
	if (area->state == ExaOffscreenAvail)
	{
	    area = area->next;
	    if (!area)
		break;
	}
	assert (area->state != ExaOffscreenAvail);
	(void) ExaOffscreenKickOut (pScreen, area);
	ExaOffscreenValidate (pScreen);
    }
    ExaOffscreenValidate (pScreen);
    ExaOffscreenFini (pScreen);
}

/** Ejects all pixmaps managed by EXA. */
static void
ExaOffscreenEjectPixmaps (ScreenPtr pScreen)
{
    ExaScreenPriv (pScreen);

    ExaOffscreenValidate (pScreen);
    /* loop until a single free area spans the space */
    for (;;)
    {
	ExaOffscreenArea *area;

	for (area = pExaScr->info->offScreenAreas; area != NULL;
	     area = area->next)
	{
	    if (area->state == ExaOffscreenRemovable &&
		area->save == exaPixmapSave)
	    {
		(void) ExaOffscreenKickOut (pScreen, area);
		ExaOffscreenValidate (pScreen);
		break;
	    }
	}
	if (area == NULL)
	    break;
    }
    ExaOffscreenValidate (pScreen);
}

static void
exaUnrealizeGlyphCaches(ScreenPtr pScreen, unsigned int format)
{
    ExaScreenPriv(pScreen);
    int i;

    for (i = 0; i < EXA_NUM_GLYPH_CACHES; i++) {
        ExaGlyphCachePtr cache = &pExaScr->glyphCaches[i];

        if (cache->format != format)
            continue;

        if (cache->picture) {
            FreePicture((pointer) cache->picture, (XID) 0);
            cache->picture = NULL;
        }

        free(cache->hashEntries);
        cache->hashEntries = NULL;

        free(cache->glyphs);
        cache->glyphs = NULL;
        cache->glyphCount = 0;
    }
}

void
exaCopyNtoN(DrawablePtr pSrcDrawable,
            DrawablePtr pDstDrawable,
            GCPtr pGC,
            BoxPtr pbox,
            int nbox,
            int dx,
            int dy,
            Bool reverse, Bool upsidedown, Pixel bitplane, void *closure)
{
    ExaScreenPriv(pDstDrawable->pScreen);

    if (pExaScr->fallback_counter ||
        (pExaScr->fallback_flags & EXA_FALLBACK_COPYWINDOW))
        return;

    if (exaHWCopyNtoN
        (pSrcDrawable, pDstDrawable, pGC, pbox, nbox, dx, dy, reverse,
         upsidedown))
        return;

    /* This is a CopyWindow, it's cleaner to fallback at the original call. */
    if (pExaScr->fallback_flags & EXA_ACCEL_COPYWINDOW) {
        pExaScr->fallback_flags |= EXA_FALLBACK_COPYWINDOW;
        return;
    }

    /* fallback */
    ExaCheckCopyNtoN(pSrcDrawable, pDstDrawable, pGC, pbox, nbox, dx, dy,
                     reverse, upsidedown, bitplane, closure);
}

/**
 * exaCreatePixmap() creates a new pixmap.
 *
 * If width and height are 0, this won't be a full-fledged pixmap and it will
 * get ModifyPixmapHeader() called on it later.  So, we mark it as pinned, because
 * ModifyPixmapHeader() would break migration.  These types of pixmaps are used
 * for scratch pixmaps, or to represent the visible screen.
 */
static PixmapPtr
exaCreatePixmap(ScreenPtr pScreen, int w, int h, int depth)
{
    PixmapPtr		pPixmap;
    ExaPixmapPrivPtr	pExaPixmap;
    int			bpp;
    ExaScreenPriv(pScreen);

    if (w > 32767 || h > 32767)
	return NullPixmap;

    pPixmap = fbCreatePixmap (pScreen, w, h, depth);
    if (!pPixmap)
	return NULL;
    pExaPixmap = ExaGetPixmapPriv(pPixmap);

    bpp = pPixmap->drawable.bitsPerPixel;

    /* Glyphs have w/h equal to zero, and may not be migrated. See exaGlyphs. */
    if (!w || !h)
	pExaPixmap->score = EXA_PIXMAP_SCORE_PINNED;
    else
	pExaPixmap->score = EXA_PIXMAP_SCORE_INIT;

    pExaPixmap->area = NULL;

    pExaPixmap->sys_ptr = pPixmap->devPrivate.ptr;
    pExaPixmap->sys_pitch = pPixmap->devKind;

    pExaPixmap->fb_ptr = NULL;
    if (pExaScr->info->flags & EXA_OFFSCREEN_ALIGN_POT && w != 1)
	pExaPixmap->fb_pitch = (1 << (exaLog2(w - 1) + 1)) * bpp / 8;
    else
	pExaPixmap->fb_pitch = w * bpp / 8;
    pExaPixmap->fb_pitch = EXA_ALIGN(pExaPixmap->fb_pitch,
				     pExaScr->info->pixmapPitchAlign);
    pExaPixmap->fb_size = pExaPixmap->fb_pitch * h;

    if (pExaPixmap->fb_pitch > 131071) {
	fbDestroyPixmap(pPixmap);
	return NULL;
    }

    /* Set up damage tracking */
    pExaPixmap->pDamage = DamageCreate (NULL, NULL, DamageReportNone, TRUE,
					pScreen, pPixmap);

    if (pExaPixmap->pDamage == NULL) {
	fbDestroyPixmap (pPixmap);
	return NULL;
    }

    DamageRegister (&pPixmap->drawable, pExaPixmap->pDamage);
    DamageSetReportAfterOp (pExaPixmap->pDamage, TRUE);

    /* None of the pixmap bits are valid initially */
    REGION_NULL(pScreen, &pExaPixmap->validReg);

    return pPixmap;
}

static void
ExaFallbackPrepareReg(DrawablePtr pDrawable,
                      GCPtr pGC,
                      int x, int y, int width, int height,
                      int index, Bool checkReads)
{
    ScreenPtr pScreen = pDrawable->pScreen;

    ExaScreenPriv(pScreen);

    if (pExaScr->prepare_access_reg &&
        !(checkReads && exaGCReadsDestination(pDrawable,
                                              pGC->planemask,
                                              pGC->fillStyle,
                                              pGC->alu, pGC->clientClipType))) {
        BoxRec box;
        RegionRec reg;
        int xoff, yoff;
        PixmapPtr pPixmap = exaGetDrawablePixmap(pDrawable);

        exaGetDrawableDeltas(pDrawable, pPixmap, &xoff, &yoff);
        box.x1 = pDrawable->x + x + xoff;
        box.y1 = pDrawable->y + y + yoff;
        box.x2 = box.x1 + width;
        box.y2 = box.y1 + height;

        RegionInit(&reg, &box, 1);
        pExaScr->prepare_access_reg(pPixmap, index, &reg);
        RegionUninit(&reg);
    }
    else
        exaPrepareAccess(pDrawable, index);
}

static void
exaPolyPoint(DrawablePtr pDrawable, GCPtr pGC, int mode, int npt,
             DDXPointPtr ppt)
{
    ExaScreenPriv(pDrawable->pScreen);
    int i;
    xRectangle *prect;

    /* If we can't reuse the current GC as is, don't bother accelerating the
     * points.
     */
    if (pExaScr->fallback_counter || pGC->fillStyle != FillSolid) {
        ExaCheckPolyPoint(pDrawable, pGC, mode, npt, ppt);
        return;
    }

    prect = malloc(sizeof(xRectangle) * npt);
    for (i = 0; i < npt; i++) {
        prect[i].x = ppt[i].x;
        prect[i].y = ppt[i].y;
        if (i > 0 && mode == CoordModePrevious) {
            prect[i].x += prect[i - 1].x;
            prect[i].y += prect[i - 1].y;
        }
        prect[i].width = 1;
        prect[i].height = 1;
    }
    pGC->ops->PolyFillRect(pDrawable, pGC, npt, prect);
    free(prect);
}

/**
 * exaFinishAccess() is EXA's wrapper for the driver's FinishAccess() handler.
 *
 * It deals with calling the driver's FinishAccess() only if necessary.
 */
void
exaFinishAccess(DrawablePtr pDrawable, int index)
{
    ScreenPtr	    pScreen = pDrawable->pScreen;
    ExaScreenPriv  (pScreen);
    PixmapPtr	    pPixmap;
    ExaPixmapPrivPtr pExaPixmap;

    pPixmap = exaGetDrawablePixmap (pDrawable);

    pExaPixmap = ExaGetPixmapPriv(pPixmap);

    /* Rehide pixmap pointer if we're doing that. */
    if (pExaPixmap != NULL && pExaScr->hideOffscreenPixmapData &&
	pExaPixmap->fb_ptr == pPixmap->devPrivate.ptr)
    {
	pPixmap->devPrivate.ptr = pExaPixmap->fb_ptr;
    }

    if (pExaScr->info->FinishAccess == NULL)
	return;

    if (!exaPixmapIsOffscreen (pPixmap))
	return;

    (*pExaScr->info->FinishAccess) (pPixmap, index);
}

/**
 * exaPrepareAccess() is EXA's wrapper for the driver's PrepareAccess() handler.
 *
 * It deals with waiting for synchronization with the card, determining if
 * PrepareAccess() is necessary, and working around PrepareAccess() failure.
 */
void
exaPrepareAccess(DrawablePtr pDrawable, int index)
{
    ScreenPtr	    pScreen = pDrawable->pScreen;
    ExaScreenPriv  (pScreen);
    PixmapPtr	    pPixmap;

    pPixmap = exaGetDrawablePixmap (pDrawable);

    if (exaPixmapIsOffscreen (pPixmap))
	exaWaitSync (pDrawable->pScreen);
    else
	return;

    /* Unhide pixmap pointer */
    if (pPixmap->devPrivate.ptr == NULL) {
	ExaPixmapPriv (pPixmap);

	pPixmap->devPrivate.ptr = pExaPixmap->fb_ptr;
    }

    if (pExaScr->info->PrepareAccess == NULL)
	return;

    if (!(*pExaScr->info->PrepareAccess) (pPixmap, index)) {
	ExaPixmapPriv (pPixmap);
	if (pExaPixmap->score != EXA_PIXMAP_SCORE_PINNED)
	    FatalError("Driver failed PrepareAccess on a pinned pixmap\n");
	exaMoveOutPixmap (pPixmap);
    }
}

static void
ExaSrcValidate(DrawablePtr pDrawable,
	       int x,
	       int y,
	       int width,
	       int height)
{
    ScreenPtr pScreen = pDrawable->pScreen;
    ExaScreenPriv(pScreen);
    PixmapPtr pPix = exaGetDrawablePixmap (pDrawable);
    BoxRec box;
    RegionRec reg;
    RegionPtr dst;
    int xoff, yoff;

    exaGetDrawableDeltas(pDrawable, pPix, &xoff, &yoff);

    box.x1 = x + xoff;
    box.y1 = y + yoff;
    box.x2 = box.x1 + width;
    box.y2 = box.y1 + height;

    dst = (pExaScr->srcPix == pPix) ? &pExaScr->srcReg :
	&pExaScr->maskReg;

    REGION_INIT(pScreen, &reg, &box, 1);
    REGION_UNION(pScreen, dst, dst, &reg);
    REGION_UNINIT(pScreen, &reg);

    if (pExaScr->SavedSourceValidate) {
        swap(pExaScr, pScreen, SourceValidate);
        pScreen->SourceValidate(pDrawable, x, y, width, height);
        swap(pExaScr, pScreen, SourceValidate);
    }
}

/**
 * exaCloseScreen() unwraps its wrapped screen functions and tears down EXA's
 * screen private, before calling down to the next CloseSccreen.
 */
static Bool
exaCloseScreen(int i, ScreenPtr pScreen)
{
    ExaScreenPriv(pScreen);
#ifdef RENDER
    PictureScreenPtr	ps = GetPictureScreenIfSet(pScreen);
#endif

    pScreen->CreateGC = pExaScr->SavedCreateGC;
    pScreen->CloseScreen = pExaScr->SavedCloseScreen;
    pScreen->GetImage = pExaScr->SavedGetImage;
    pScreen->GetSpans = pExaScr->SavedGetSpans;
    pScreen->PaintWindowBackground = pExaScr->SavedPaintWindowBackground;
    pScreen->PaintWindowBorder = pExaScr->SavedPaintWindowBorder;
    pScreen->CreatePixmap = pExaScr->SavedCreatePixmap;
    pScreen->DestroyPixmap = pExaScr->SavedDestroyPixmap;
    pScreen->CopyWindow = pExaScr->SavedCopyWindow;
#ifdef RENDER
    if (ps) {
	ps->Composite = pExaScr->SavedComposite;
	ps->Glyphs = pExaScr->SavedGlyphs;
	ps->Trapezoids = pExaScr->SavedTrapezoids;
    }
#endif

    xfree (pExaScr);

    return (*pScreen->CloseScreen) (i, pScreen);
}

/**
 * exaOffscreenInit initializes the offscreen memory manager.
 *
 * @param pScreen current screen
 *
 * exaOffscreenInit is called by exaDriverInit to set up the memory manager for
 * the screen, if any offscreen memory is available.
 */
Bool
exaOffscreenInit (ScreenPtr pScreen)
{
    ExaScreenPriv (pScreen);
    ExaOffscreenArea *area;

    /* Allocate a big free area */
    area = xalloc (sizeof (ExaOffscreenArea));

    if (!area)
	return FALSE;

    area->state = ExaOffscreenAvail;
    area->base_offset = pExaScr->info->offScreenBase;
    area->offset = area->base_offset;
    area->size = pExaScr->info->memorySize - area->base_offset;
    area->save = NULL;
    area->next = NULL;
    area->score = 0;

    /* Add it to the free areas */
    pExaScr->info->offScreenAreas = area;

    ExaOffscreenValidate (pScreen);

    return TRUE;
}

Bool
exaDestroyPixmap_classic(PixmapPtr pPixmap)
{
    ScreenPtr pScreen = pPixmap->drawable.pScreen;

    ExaScreenPriv(pScreen);
    Bool ret;

    if (pPixmap->refcnt == 1) {
        ExaPixmapPriv(pPixmap);

        exaDestroyPixmap(pPixmap);

        if (pExaPixmap->area) {
            DBG_PIXMAP(("-- 0x%p (0x%x) (%dx%d)\n",
                        (void *) pPixmap->drawable.id,
                        ExaGetPixmapPriv(pPixmap)->area->offset,
                        pPixmap->drawable.width, pPixmap->drawable.height));
            /* Free the offscreen area */
            exaOffscreenFree(pPixmap->drawable.pScreen, pExaPixmap->area);
            pPixmap->devPrivate.ptr = pExaPixmap->sys_ptr;
            pPixmap->devKind = pExaPixmap->sys_pitch;
        }
        RegionUninit(&pExaPixmap->validSys);
        RegionUninit(&pExaPixmap->validFB);
    }

    swap(pExaScr, pScreen, DestroyPixmap);
    ret = pScreen->DestroyPixmap(pPixmap);
    swap(pExaScr, pScreen, DestroyPixmap);

    return ret;
}

/**
 * If the pixmap is currently dirty, this copies at least the dirty area from
 * the system memory copy to the framebuffer memory copy.  Both areas must be
 * allocated.
 */
static void
exaCopyDirtyToFb (PixmapPtr pPixmap)
{
    ExaScreenPriv (pPixmap->drawable.pScreen);
    ExaPixmapPriv (pPixmap);
    RegionPtr pRegion = DamageRegion (pExaPixmap->pDamage);
    CARD8 *save_ptr;
    int save_pitch;
    BoxPtr pBox = REGION_RECTS(pRegion);
    int nbox = REGION_NUM_RECTS(pRegion);
    Bool do_sync = FALSE;

    save_ptr = pPixmap->devPrivate.ptr;
    save_pitch = pPixmap->devKind;
    pPixmap->devPrivate.ptr = pExaPixmap->fb_ptr;
    pPixmap->devKind = pExaPixmap->fb_pitch;

    while (nbox--) {
	pBox->x1 = max(pBox->x1, 0);
	pBox->y1 = max(pBox->y1, 0);
	pBox->x2 = min(pBox->x2, pPixmap->drawable.width);
	pBox->y2 = min(pBox->y2, pPixmap->drawable.height);

	if (pBox->x1 >= pBox->x2 || pBox->y1 >= pBox->y2)
	    continue;

	if (pExaScr->info->UploadToScreen == NULL ||
	    !pExaScr->info->UploadToScreen (pPixmap,
					    pBox->x1, pBox->y1,
					    pBox->x2 - pBox->x1,
					    pBox->y2 - pBox->y1,
					    pExaPixmap->sys_ptr
					    + pBox->y1 * pExaPixmap->sys_pitch
					    + pBox->x1 * pPixmap->drawable.bitsPerPixel / 8,
					    pExaPixmap->sys_pitch))
	{
	    exaPrepareAccess(&pPixmap->drawable, EXA_PREPARE_DEST);
	    exaMemcpyBox (pPixmap, pBox,
			  pExaPixmap->sys_ptr, pExaPixmap->sys_pitch,
			  pExaPixmap->fb_ptr, pExaPixmap->fb_pitch);
	    exaFinishAccess(&pPixmap->drawable, EXA_PREPARE_DEST);
	}
	else
	    do_sync = TRUE;

	pBox++;
    }

    if (do_sync)
	exaMarkSync (pPixmap->drawable.pScreen);

    pPixmap->devPrivate.ptr = save_ptr;
    pPixmap->devKind = save_pitch;

    /* The previously damaged bits are now no longer damaged but valid */
    REGION_UNION(pPixmap->drawable.pScreen,
		 &pExaPixmap->validReg, &pExaPixmap->validReg, pRegion);
    DamageEmpty (pExaPixmap->pDamage);
}

/**
 * exaDDXDriverInit is required by the top-level EXA module, and is used by
 * the xorg DDX to hook in its EnableDisableFB wrapper.  We don't need it, since
 * we won't be enabling/disabling the FB.
 */
void
exaDDXDriverInit(ScreenPtr pScreen)
{
    ExaScreenPriv(pScreen);

    pExaScr->migration = ExaMigrationSmart;
    pExaScr->checkDirtyCorrectness = TRUE;
}

/**
 * exaWaitSync() ensures that all drawing has been completed.
 *
 * @param pScreen screen being synchronized.
 *
 * Calls down into the driver to ensure that all previous drawing has completed.
 * It should always be called before relying on the framebuffer contents
 * reflecting previous drawing, from a CPU perspective.
 */
void exaWaitSync(ScreenPtr pScreen)
{
    ExaScreenPriv(pScreen);

    if (pExaScr->info->needsSync && !pExaScr->swappedOut) {
        (*pExaScr->info->WaitMarker)(pScreen, pExaScr->info->lastMarker);
        pExaScr->info->needsSync = FALSE;
    }
}

/**
 * exaMarkSync() should be called after any asynchronous drawing by the hardware.
 *
 * @param pScreen screen which drawing occurred on
 *
 * exaMarkSync() sets a flag to indicate that some asynchronous drawing has
 * happened and a WaitSync() will be necessary before relying on the contents of
 * offscreen memory from the CPU's perspective.  It also calls an optional
 * driver MarkSync() callback, the return value of which may be used to do partial
 * synchronization with the hardware in the future.
 */
void exaMarkSync(ScreenPtr pScreen)
{
    ExaScreenPriv(pScreen);

    pExaScr->info->needsSync = TRUE;
    if (pExaScr->info->MarkSync != NULL) {
        pExaScr->info->lastMarker = (*pExaScr->info->MarkSync)(pScreen);
    }
}

/**
 * Allocates a framebuffer copy of the pixmap if necessary, and then copies
 * any necessary pixmap data into the framebuffer copy and points the pixmap at
 * it.
 *
 * Note that when first allocated, a pixmap will have FALSE dirty flag.
 * This is intentional because pixmap data starts out undefined.  So if we move
 * it in due to the first operation against it being accelerated, it will have
 * undefined framebuffer contents that we didn't have to upload.  If we do
 * moveouts (and moveins) after the first movein, then we will only have to copy
 * back and forth if the pixmap was written to after the last synchronization of
 * the two copies.  Then, at exaPixmapSave (when the framebuffer copy goes away)
 * we mark the pixmap dirty, so that the next exaMoveInPixmap will actually move
 * all the data, since it's almost surely all valid now.
 */
static void
exaDoMoveInPixmap(ExaMigrationPtr migrate)
{
    PixmapPtr pPixmap = migrate->pPix;
    ScreenPtr pScreen = pPixmap->drawable.pScreen;

    ExaScreenPriv(pScreen);
    ExaPixmapPriv(pPixmap);

    /* If we're VT-switched away, no touching card memory allowed. */
    if (pExaScr->swappedOut)
        return;

    /* If we're not allowed to move, then fail. */
    if (exaPixmapIsPinned(pPixmap))
        return;

    /* Don't migrate in pixmaps which are less than 8bpp.  This avoids a lot of
     * fragility in EXA, and <8bpp is probably not used enough any more to care
     * (at least, not in acceleratd paths).
     */
    if (pPixmap->drawable.bitsPerPixel < 8)
        return;

    if (pExaPixmap->accel_blocked)
        return;

    if (pExaPixmap->area == NULL) {
        pExaPixmap->area =
            exaOffscreenAlloc(pScreen, pExaPixmap->fb_size,
                              pExaScr->info->pixmapOffsetAlign, FALSE,
                              exaPixmapSave, (pointer) pPixmap);
        if (pExaPixmap->area == NULL)
            return;

        pExaPixmap->fb_ptr = (CARD8 *) pExaScr->info->memoryBase +
                             pExaPixmap->area->offset;
    }

    exaCopyDirtyToFb(migrate);

    if (exaPixmapHasGpuCopy(pPixmap))
        return;

    DBG_MIGRATE(("-> %p (0x%x) (%dx%d) (%c)\n", pPixmap,
                 (ExaGetPixmapPriv(pPixmap)->area ?
                  ExaGetPixmapPriv(pPixmap)->area->offset : 0),
                 pPixmap->drawable.width,
                 pPixmap->drawable.height,
                 exaPixmapIsDirty(pPixmap) ? 'd' : 'c'));

    pExaPixmap->use_gpu_copy = TRUE;

    pPixmap->devKind = pExaPixmap->fb_pitch;
    pPixmap->drawable.serialNumber = NEXT_SERIAL_NUMBER;
}

/**
 * If the pixmap is currently dirty, this copies at least the dirty area from
 * the system memory copy to the framebuffer memory copy.  Both areas must be
 * allocated.
 */
void
exaCopyDirtyToFb(ExaMigrationPtr migrate)
{
    PixmapPtr pPixmap = migrate->pPix;

    ExaScreenPriv(pPixmap->drawable.pScreen);
    ExaPixmapPriv(pPixmap);

    exaCopyDirty(migrate, &pExaPixmap->validFB, &pExaPixmap->validSys,
                 pExaScr->info->UploadToScreen, EXA_PREPARE_DEST, NULL);
}