diff options
| -rw-r--r-- | drivers/gpu/drm/radeon/evergreen.c | 8 | ||||
| -rw-r--r-- | drivers/gpu/drm/radeon/ni.c | 20 | ||||
| -rw-r--r-- | drivers/gpu/drm/radeon/r100.c | 8 | ||||
| -rw-r--r-- | drivers/gpu/drm/radeon/r300.c | 8 | ||||
| -rw-r--r-- | drivers/gpu/drm/radeon/r420.c | 8 | ||||
| -rw-r--r-- | drivers/gpu/drm/radeon/r520.c | 8 | ||||
| -rw-r--r-- | drivers/gpu/drm/radeon/r600.c | 11 | ||||
| -rw-r--r-- | drivers/gpu/drm/radeon/radeon.h | 5 | ||||
| -rw-r--r-- | drivers/gpu/drm/radeon/radeon_fence.c | 89 | ||||
| -rw-r--r-- | drivers/gpu/drm/radeon/rs400.c | 8 | ||||
| -rw-r--r-- | drivers/gpu/drm/radeon/rs600.c | 8 | ||||
| -rw-r--r-- | drivers/gpu/drm/radeon/rs690.c | 8 | ||||
| -rw-r--r-- | drivers/gpu/drm/radeon/rv515.c | 8 | ||||
| -rw-r--r-- | drivers/gpu/drm/radeon/rv770.c | 8 |
14 files changed, 152 insertions, 53 deletions
diff --git a/drivers/gpu/drm/radeon/evergreen.c b/drivers/gpu/drm/radeon/evergreen.c index b19ace86121e..ba19b9a697b0 100644 --- a/drivers/gpu/drm/radeon/evergreen.c +++ b/drivers/gpu/drm/radeon/evergreen.c @@ -3111,6 +3111,12 @@ static int evergreen_startup(struct radeon_device *rdev) if (r) return r; + r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX); + if (r) { + dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r); + return r; + } + /* Enable IRQ */ r = r600_irq_init(rdev); if (r) { @@ -3229,7 +3235,7 @@ int evergreen_init(struct radeon_device *rdev) /* Initialize clocks */ radeon_get_clock_info(rdev->ddev); /* Fence driver */ - r = radeon_fence_driver_init(rdev, 1); + r = radeon_fence_driver_init(rdev); if (r) return r; /* initialize AGP */ diff --git a/drivers/gpu/drm/radeon/ni.c b/drivers/gpu/drm/radeon/ni.c index 022a606c783b..a9b0e615804f 100644 --- a/drivers/gpu/drm/radeon/ni.c +++ b/drivers/gpu/drm/radeon/ni.c @@ -1384,6 +1384,24 @@ static int cayman_startup(struct radeon_device *rdev) if (r) return r; + r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX); + if (r) { + dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r); + return r; + } + + r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX); + if (r) { + dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r); + return r; + } + + r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX); + if (r) { + dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r); + return r; + } + /* Enable IRQ */ r = r600_irq_init(rdev); if (r) { @@ -1492,7 +1510,7 @@ int cayman_init(struct radeon_device *rdev) /* Initialize clocks */ radeon_get_clock_info(rdev->ddev); /* Fence driver */ - r = radeon_fence_driver_init(rdev, 3); + r = radeon_fence_driver_init(rdev); if (r) return r; /* initialize memory controller */ diff --git a/drivers/gpu/drm/radeon/r100.c b/drivers/gpu/drm/radeon/r100.c index a40e893a7b69..bed56c7b690b 100644 --- a/drivers/gpu/drm/radeon/r100.c +++ b/drivers/gpu/drm/radeon/r100.c @@ -3916,6 +3916,12 @@ static int r100_startup(struct radeon_device *rdev) if (r) return r; + r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX); + if (r) { + dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r); + return r; + } + /* Enable IRQ */ r100_irq_set(rdev); rdev->config.r100.hdp_cntl = RREG32(RADEON_HOST_PATH_CNTL); @@ -4059,7 +4065,7 @@ int r100_init(struct radeon_device *rdev) /* initialize VRAM */ r100_mc_init(rdev); /* Fence driver */ - r = radeon_fence_driver_init(rdev, 1); + r = radeon_fence_driver_init(rdev); if (r) return r; r = radeon_irq_kms_init(rdev); diff --git a/drivers/gpu/drm/radeon/r300.c b/drivers/gpu/drm/radeon/r300.c index 6a96b31b558f..e2dfae4b40e6 100644 --- a/drivers/gpu/drm/radeon/r300.c +++ b/drivers/gpu/drm/radeon/r300.c @@ -1399,6 +1399,12 @@ static int r300_startup(struct radeon_device *rdev) if (r) return r; + r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX); + if (r) { + dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r); + return r; + } + /* Enable IRQ */ r100_irq_set(rdev); rdev->config.r300.hdp_cntl = RREG32(RADEON_HOST_PATH_CNTL); @@ -1521,7 +1527,7 @@ int r300_init(struct radeon_device *rdev) /* initialize memory controller */ r300_mc_init(rdev); /* Fence driver */ - r = radeon_fence_driver_init(rdev, 1); + r = radeon_fence_driver_init(rdev); if (r) return r; r = radeon_irq_kms_init(rdev); diff --git a/drivers/gpu/drm/radeon/r420.c b/drivers/gpu/drm/radeon/r420.c index 1d3231f3a090..17ecff1000e7 100644 --- a/drivers/gpu/drm/radeon/r420.c +++ b/drivers/gpu/drm/radeon/r420.c @@ -258,6 +258,12 @@ static int r420_startup(struct radeon_device *rdev) if (r) return r; + r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX); + if (r) { + dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r); + return r; + } + /* Enable IRQ */ r100_irq_set(rdev); rdev->config.r300.hdp_cntl = RREG32(RADEON_HOST_PATH_CNTL); @@ -391,7 +397,7 @@ int r420_init(struct radeon_device *rdev) r300_mc_init(rdev); r420_debugfs(rdev); /* Fence driver */ - r = radeon_fence_driver_init(rdev, 1); + r = radeon_fence_driver_init(rdev); if (r) { return r; } diff --git a/drivers/gpu/drm/radeon/r520.c b/drivers/gpu/drm/radeon/r520.c index cb96a51f7f7c..39b31544c6e7 100644 --- a/drivers/gpu/drm/radeon/r520.c +++ b/drivers/gpu/drm/radeon/r520.c @@ -187,6 +187,12 @@ static int r520_startup(struct radeon_device *rdev) if (r) return r; + r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX); + if (r) { + dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r); + return r; + } + /* Enable IRQ */ rs600_irq_set(rdev); rdev->config.r300.hdp_cntl = RREG32(RADEON_HOST_PATH_CNTL); @@ -278,7 +284,7 @@ int r520_init(struct radeon_device *rdev) r520_mc_init(rdev); rv515_debugfs(rdev); /* Fence driver */ - r = radeon_fence_driver_init(rdev, 1); + r = radeon_fence_driver_init(rdev); if (r) return r; r = radeon_irq_kms_init(rdev); diff --git a/drivers/gpu/drm/radeon/r600.c b/drivers/gpu/drm/radeon/r600.c index 17538926cc47..f2deadfcd88f 100644 --- a/drivers/gpu/drm/radeon/r600.c +++ b/drivers/gpu/drm/radeon/r600.c @@ -2313,8 +2313,7 @@ void r600_fence_ring_emit(struct radeon_device *rdev, struct radeon_ring *ring = &rdev->ring[fence->ring]; if (rdev->wb.use_event) { - u64 addr = rdev->wb.gpu_addr + R600_WB_EVENT_OFFSET + - (u64)(rdev->fence_drv[fence->ring].scratch_reg - rdev->scratch.reg_base); + u64 addr = rdev->fence_drv[fence->ring].gpu_addr; /* flush read cache over gart */ radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3)); radeon_ring_write(ring, PACKET3_TC_ACTION_ENA | @@ -2459,6 +2458,12 @@ int r600_startup(struct radeon_device *rdev) if (r) return r; + r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX); + if (r) { + dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r); + return r; + } + /* Enable IRQ */ r = r600_irq_init(rdev); if (r) { @@ -2589,7 +2594,7 @@ int r600_init(struct radeon_device *rdev) /* Initialize clocks */ radeon_get_clock_info(rdev->ddev); /* Fence driver */ - r = radeon_fence_driver_init(rdev, 1); + r = radeon_fence_driver_init(rdev); if (r) return r; if (rdev->flags & RADEON_IS_AGP) { diff --git a/drivers/gpu/drm/radeon/radeon.h b/drivers/gpu/drm/radeon/radeon.h index 014065af60dc..5777afb361de 100644 --- a/drivers/gpu/drm/radeon/radeon.h +++ b/drivers/gpu/drm/radeon/radeon.h @@ -192,6 +192,8 @@ extern int sumo_get_temp(struct radeon_device *rdev); */ struct radeon_fence_driver { uint32_t scratch_reg; + uint64_t gpu_addr; + volatile uint32_t *cpu_addr; atomic_t seq; uint32_t last_seq; unsigned long last_jiffies; @@ -215,7 +217,8 @@ struct radeon_fence { int ring; }; -int radeon_fence_driver_init(struct radeon_device *rdev, int num_rings); +int radeon_fence_driver_start_ring(struct radeon_device *rdev, int ring); +int radeon_fence_driver_init(struct radeon_device *rdev); void radeon_fence_driver_fini(struct radeon_device *rdev); int radeon_fence_create(struct radeon_device *rdev, struct radeon_fence **fence, int ring); int radeon_fence_emit(struct radeon_device *rdev, struct radeon_fence *fence); diff --git a/drivers/gpu/drm/radeon/radeon_fence.c b/drivers/gpu/drm/radeon/radeon_fence.c index 809e66e8a86e..8d626baaa064 100644 --- a/drivers/gpu/drm/radeon/radeon_fence.c +++ b/drivers/gpu/drm/radeon/radeon_fence.c @@ -42,35 +42,22 @@ static void radeon_fence_write(struct radeon_device *rdev, u32 seq, int ring) { - u32 scratch_index; - if (rdev->wb.enabled) { - if (rdev->wb.use_event) - scratch_index = R600_WB_EVENT_OFFSET + - rdev->fence_drv[ring].scratch_reg - rdev->scratch.reg_base; - else - scratch_index = RADEON_WB_SCRATCH_OFFSET + - rdev->fence_drv[ring].scratch_reg - rdev->scratch.reg_base; - rdev->wb.wb[scratch_index/4] = cpu_to_le32(seq); - } else + *rdev->fence_drv[ring].cpu_addr = cpu_to_le32(seq); + } else { WREG32(rdev->fence_drv[ring].scratch_reg, seq); + } } static u32 radeon_fence_read(struct radeon_device *rdev, int ring) { u32 seq = 0; - u32 scratch_index; if (rdev->wb.enabled) { - if (rdev->wb.use_event) - scratch_index = R600_WB_EVENT_OFFSET + - rdev->fence_drv[ring].scratch_reg - rdev->scratch.reg_base; - else - scratch_index = RADEON_WB_SCRATCH_OFFSET + - rdev->fence_drv[ring].scratch_reg - rdev->scratch.reg_base; - seq = le32_to_cpu(rdev->wb.wb[scratch_index/4]); - } else + seq = le32_to_cpu(*rdev->fence_drv[ring].cpu_addr); + } else { seq = RREG32(rdev->fence_drv[ring].scratch_reg); + } return seq; } @@ -389,36 +376,61 @@ int radeon_fence_count_emitted(struct radeon_device *rdev, int ring) return not_processed; } -int radeon_fence_driver_init(struct radeon_device *rdev, int num_rings) +int radeon_fence_driver_start_ring(struct radeon_device *rdev, int ring) { unsigned long irq_flags; - int r, ring; + uint64_t index; + int r; - for (ring = 0; ring < num_rings; ring++) { - write_lock_irqsave(&rdev->fence_lock, irq_flags); + write_lock_irqsave(&rdev->fence_lock, irq_flags); + radeon_scratch_free(rdev, rdev->fence_drv[ring].scratch_reg); + if (rdev->wb.use_event) { + rdev->fence_drv[ring].scratch_reg = 0; + index = R600_WB_EVENT_OFFSET + ring * 4; + } else { r = radeon_scratch_get(rdev, &rdev->fence_drv[ring].scratch_reg); if (r) { dev_err(rdev->dev, "fence failed to get scratch register\n"); write_unlock_irqrestore(&rdev->fence_lock, irq_flags); return r; } - radeon_fence_write(rdev, 0, ring); - atomic_set(&rdev->fence_drv[ring].seq, 0); - INIT_LIST_HEAD(&rdev->fence_drv[ring].created); - INIT_LIST_HEAD(&rdev->fence_drv[ring].emitted); - INIT_LIST_HEAD(&rdev->fence_drv[ring].signaled); - init_waitqueue_head(&rdev->fence_drv[ring].queue); - rdev->fence_drv[ring].initialized = true; - write_unlock_irqrestore(&rdev->fence_lock, irq_flags); + index = RADEON_WB_SCRATCH_OFFSET + + rdev->fence_drv[ring].scratch_reg - + rdev->scratch.reg_base; } - for (ring = num_rings; ring < RADEON_NUM_RINGS; ring++) { - write_lock_irqsave(&rdev->fence_lock, irq_flags); - INIT_LIST_HEAD(&rdev->fence_drv[ring].created); - INIT_LIST_HEAD(&rdev->fence_drv[ring].emitted); - INIT_LIST_HEAD(&rdev->fence_drv[ring].signaled); - rdev->fence_drv[ring].initialized = false; - write_unlock_irqrestore(&rdev->fence_lock, irq_flags); + rdev->fence_drv[ring].cpu_addr = &rdev->wb.wb[index/4]; + rdev->fence_drv[ring].gpu_addr = rdev->wb.gpu_addr + index; + radeon_fence_write(rdev, atomic_read(&rdev->fence_drv[ring].seq), ring); + rdev->fence_drv[ring].initialized = true; + DRM_INFO("fence driver on ring %d use gpu addr 0x%08Lx and cpu addr 0x%p\n", + ring, rdev->fence_drv[ring].gpu_addr, rdev->fence_drv[ring].cpu_addr); + write_unlock_irqrestore(&rdev->fence_lock, irq_flags); + return 0; +} + +static void radeon_fence_driver_init_ring(struct radeon_device *rdev, int ring) +{ + rdev->fence_drv[ring].scratch_reg = -1; + rdev->fence_drv[ring].cpu_addr = NULL; + rdev->fence_drv[ring].gpu_addr = 0; + atomic_set(&rdev->fence_drv[ring].seq, 0); + INIT_LIST_HEAD(&rdev->fence_drv[ring].created); + INIT_LIST_HEAD(&rdev->fence_drv[ring].emitted); + INIT_LIST_HEAD(&rdev->fence_drv[ring].signaled); + init_waitqueue_head(&rdev->fence_drv[ring].queue); + rdev->fence_drv[ring].initialized = false; +} + +int radeon_fence_driver_init(struct radeon_device *rdev) +{ + unsigned long irq_flags; + int ring; + + write_lock_irqsave(&rdev->fence_lock, irq_flags); + for (ring = 0; ring < RADEON_NUM_RINGS; ring++) { + radeon_fence_driver_init_ring(rdev, ring); } + write_unlock_irqrestore(&rdev->fence_lock, irq_flags); if (radeon_debugfs_fence_init(rdev)) { dev_err(rdev->dev, "fence debugfs file creation failed\n"); } @@ -433,6 +445,7 @@ void radeon_fence_driver_fini(struct radeon_device *rdev) for (ring = 0; ring < RADEON_NUM_RINGS; ring++) { if (!rdev->fence_drv[ring].initialized) continue; + radeon_fence_wait_last(rdev, ring); wake_up_all(&rdev->fence_drv[ring].queue); write_lock_irqsave(&rdev->fence_lock, irq_flags); radeon_scratch_free(rdev, rdev->fence_drv[ring].scratch_reg); diff --git a/drivers/gpu/drm/radeon/rs400.c b/drivers/gpu/drm/radeon/rs400.c index c71fa16106ca..5c86d5161fdb 100644 --- a/drivers/gpu/drm/radeon/rs400.c +++ b/drivers/gpu/drm/radeon/rs400.c @@ -410,6 +410,12 @@ static int rs400_startup(struct radeon_device *rdev) if (r) return r; + r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX); + if (r) { + dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r); + return r; + } + /* Enable IRQ */ r100_irq_set(rdev); rdev->config.r300.hdp_cntl = RREG32(RADEON_HOST_PATH_CNTL); @@ -516,7 +522,7 @@ int rs400_init(struct radeon_device *rdev) /* initialize memory controller */ rs400_mc_init(rdev); /* Fence driver */ - r = radeon_fence_driver_init(rdev, 1); + r = radeon_fence_driver_init(rdev); if (r) return r; r = radeon_irq_kms_init(rdev); diff --git a/drivers/gpu/drm/radeon/rs600.c b/drivers/gpu/drm/radeon/rs600.c index 1c9ab9409531..3fe384741fc0 100644 --- a/drivers/gpu/drm/radeon/rs600.c +++ b/drivers/gpu/drm/radeon/rs600.c @@ -849,6 +849,12 @@ static int rs600_startup(struct radeon_device *rdev) if (r) return r; + r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX); + if (r) { + dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r); + return r; + } + /* Enable IRQ */ rs600_irq_set(rdev); rdev->config.r300.hdp_cntl = RREG32(RADEON_HOST_PATH_CNTL); @@ -962,7 +968,7 @@ int rs600_init(struct radeon_device *rdev) rs600_mc_init(rdev); rs600_debugfs(rdev); /* Fence driver */ - r = radeon_fence_driver_init(rdev, 1); + r = radeon_fence_driver_init(rdev); if (r) return r; r = radeon_irq_kms_init(rdev); diff --git a/drivers/gpu/drm/radeon/rs690.c b/drivers/gpu/drm/radeon/rs690.c index 8aa5e7ef2efc..ae941d196d3f 100644 --- a/drivers/gpu/drm/radeon/rs690.c +++ b/drivers/gpu/drm/radeon/rs690.c @@ -621,6 +621,12 @@ static int rs690_startup(struct radeon_device *rdev) if (r) return r; + r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX); + if (r) { + dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r); + return r; + } + /* Enable IRQ */ rs600_irq_set(rdev); rdev->config.r300.hdp_cntl = RREG32(RADEON_HOST_PATH_CNTL); @@ -735,7 +741,7 @@ int rs690_init(struct radeon_device *rdev) rs690_mc_init(rdev); rv515_debugfs(rdev); /* Fence driver */ - r = radeon_fence_driver_init(rdev, 1); + r = radeon_fence_driver_init(rdev); if (r) return r; r = radeon_irq_kms_init(rdev); diff --git a/drivers/gpu/drm/radeon/rv515.c b/drivers/gpu/drm/radeon/rv515.c index beed57c7df96..21d90d9fe11c 100644 --- a/drivers/gpu/drm/radeon/rv515.c +++ b/drivers/gpu/drm/radeon/rv515.c @@ -393,6 +393,12 @@ static int rv515_startup(struct radeon_device *rdev) if (r) return r; + r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX); + if (r) { + dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r); + return r; + } + /* Enable IRQ */ rs600_irq_set(rdev); rdev->config.r300.hdp_cntl = RREG32(RADEON_HOST_PATH_CNTL); @@ -511,7 +517,7 @@ int rv515_init(struct radeon_device *rdev) rv515_mc_init(rdev); rv515_debugfs(rdev); /* Fence driver */ - r = radeon_fence_driver_init(rdev, 1); + r = radeon_fence_driver_init(rdev); if (r) return r; r = radeon_irq_kms_init(rdev); diff --git a/drivers/gpu/drm/radeon/rv770.c b/drivers/gpu/drm/radeon/rv770.c index f01603d522bb..0a1283d24027 100644 --- a/drivers/gpu/drm/radeon/rv770.c +++ b/drivers/gpu/drm/radeon/rv770.c @@ -1083,6 +1083,12 @@ static int rv770_startup(struct radeon_device *rdev) if (r) return r; + r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX); + if (r) { + dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r); + return r; + } + /* Enable IRQ */ r = r600_irq_init(rdev); if (r) { @@ -1196,7 +1202,7 @@ int rv770_init(struct radeon_device *rdev) /* Initialize clocks */ radeon_get_clock_info(rdev->ddev); /* Fence driver */ - r = radeon_fence_driver_init(rdev, 1); + r = radeon_fence_driver_init(rdev); if (r) return r; /* initialize AGP */ |