1 // SPDX-License-Identifier: GPL-2.0
4  * Copyright 2016-2021 HabanaLabs, Ltd.
15 	struct hl_encaps_signals_mgr *mgr = &handle->ctx->sig_mgr;  in encaps_handle_do_release()
18 		hw_sob_put(handle->hw_sob);  in encaps_handle_do_release()
20 	spin_lock(&mgr->lock);  in encaps_handle_do_release()
21 	idr_remove(&mgr->handles, handle->id);  in encaps_handle_do_release()
22 	spin_unlock(&mgr->lock);  in encaps_handle_do_release()
25 		hl_ctx_put(handle->ctx);  in encaps_handle_do_release()
56 	spin_lock_init(&mgr->lock);  in hl_encaps_sig_mgr_init()
57 	idr_init(&mgr->handles);  in hl_encaps_sig_mgr_init()
66 	idp = &mgr->handles;  in hl_encaps_sig_mgr_fini()
69 	 * released as part of CS roll-back.  in hl_encaps_sig_mgr_fini()
72 		dev_warn(hdev->dev,  in hl_encaps_sig_mgr_fini()
75 			kref_put(&handle->refcount, hl_encaps_release_handle_and_put_sob);  in hl_encaps_sig_mgr_fini()
78 	idr_destroy(&mgr->handles);  in hl_encaps_sig_mgr_fini()
81 static void hl_ctx_fini(struct hl_ctx *ctx)  in hl_ctx_fini()  argument
83 	struct hl_device *hdev = ctx->hdev;  in hl_ctx_fini()
89 	hl_hw_block_mem_fini(ctx);  in hl_ctx_fini()
99 	for (i = 0 ; i < hdev->asic_prop.max_pending_cs ; i++)  in hl_ctx_fini()
100 		hl_fence_put(ctx->cs_pending[i]);  in hl_ctx_fini()
102 	kfree(ctx->cs_pending);  in hl_ctx_fini()
104 	if (ctx->asid != HL_KERNEL_ASID_ID) {  in hl_ctx_fini()
105 		dev_dbg(hdev->dev, "closing user context, asid=%u\n", ctx->asid);  in hl_ctx_fini()
111 		if (hdev->in_debug)  in hl_ctx_fini()
112 			hl_device_set_debug_mode(hdev, ctx, false);  in hl_ctx_fini()
114 		hdev->asic_funcs->ctx_fini(ctx);  in hl_ctx_fini()
116 		hl_dec_ctx_fini(ctx);  in hl_ctx_fini()
118 		hl_cb_va_pool_fini(ctx);  in hl_ctx_fini()
119 		hl_vm_ctx_fini(ctx);  in hl_ctx_fini()
120 		hl_asid_free(hdev, ctx->asid);  in hl_ctx_fini()
121 		hl_encaps_sig_mgr_fini(hdev, &ctx->sig_mgr);  in hl_ctx_fini()
122 		mutex_destroy(&ctx->ts_reg_lock);  in hl_ctx_fini()
124 		dev_dbg(hdev->dev, "closing kernel context\n");  in hl_ctx_fini()
125 		hdev->asic_funcs->ctx_fini(ctx);  in hl_ctx_fini()
126 		hl_vm_ctx_fini(ctx);  in hl_ctx_fini()
127 		hl_mmu_ctx_fini(ctx);  in hl_ctx_fini()
133 	struct hl_ctx *ctx;  in hl_ctx_do_release()  local
135 	ctx = container_of(ref, struct hl_ctx, refcount);  in hl_ctx_do_release()
137 	hl_ctx_fini(ctx);  in hl_ctx_do_release()
139 	if (ctx->hpriv) {  in hl_ctx_do_release()
140 		struct hl_fpriv *hpriv = ctx->hpriv;  in hl_ctx_do_release()
142 		mutex_lock(&hpriv->ctx_lock);  in hl_ctx_do_release()
143 		hpriv->ctx = NULL;  in hl_ctx_do_release()
144 		mutex_unlock(&hpriv->ctx_lock);  in hl_ctx_do_release()
149 	kfree(ctx);  in hl_ctx_do_release()
154 	struct hl_ctx_mgr *ctx_mgr = &hpriv->ctx_mgr;  in hl_ctx_create()
155 	struct hl_ctx *ctx;  in hl_ctx_create()  local
158 	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);  in hl_ctx_create()
159 	if (!ctx) {  in hl_ctx_create()
160 		rc = -ENOMEM;  in hl_ctx_create()
164 	mutex_lock(&ctx_mgr->lock);  in hl_ctx_create()
165 	rc = idr_alloc(&ctx_mgr->handles, ctx, 1, 0, GFP_KERNEL);  in hl_ctx_create()
166 	mutex_unlock(&ctx_mgr->lock);  in hl_ctx_create()
169 		dev_err(hdev->dev, "Failed to allocate IDR for a new CTX\n");  in hl_ctx_create()
173 	ctx->handle = rc;  in hl_ctx_create()
175 	rc = hl_ctx_init(hdev, ctx, false);  in hl_ctx_create()
180 	ctx->hpriv = hpriv;  in hl_ctx_create()
183 	hpriv->ctx = ctx;  in hl_ctx_create()
186 	hdev->is_compute_ctx_active = true;  in hl_ctx_create()
191 	mutex_lock(&ctx_mgr->lock);  in hl_ctx_create()
192 	idr_remove(&ctx_mgr->handles, ctx->handle);  in hl_ctx_create()
193 	mutex_unlock(&ctx_mgr->lock);  in hl_ctx_create()
195 	kfree(ctx);  in hl_ctx_create()
200 int hl_ctx_init(struct hl_device *hdev, struct hl_ctx *ctx, bool is_kernel_ctx)  in hl_ctx_init()  argument
204 	ctx->hdev = hdev;  in hl_ctx_init()
206 	kref_init(&ctx->refcount);  in hl_ctx_init()
208 	ctx->cs_sequence = 1;  in hl_ctx_init()
209 	spin_lock_init(&ctx->cs_lock);  in hl_ctx_init()
210 	atomic_set(&ctx->thread_ctx_switch_token, 1);  in hl_ctx_init()
211 	ctx->thread_ctx_switch_wait_token = 0;  in hl_ctx_init()
212 	ctx->cs_pending = kcalloc(hdev->asic_prop.max_pending_cs,  in hl_ctx_init()
215 	if (!ctx->cs_pending)  in hl_ctx_init()
216 		return -ENOMEM;  in hl_ctx_init()
218 	INIT_LIST_HEAD(&ctx->outcome_store.used_list);  in hl_ctx_init()
219 	INIT_LIST_HEAD(&ctx->outcome_store.free_list);  in hl_ctx_init()
220 	hash_init(ctx->outcome_store.outcome_map);  in hl_ctx_init()
221 	for (i = 0; i < ARRAY_SIZE(ctx->outcome_store.nodes_pool); ++i)  in hl_ctx_init()
222 		list_add(&ctx->outcome_store.nodes_pool[i].list_link,  in hl_ctx_init()
223 			 &ctx->outcome_store.free_list);  in hl_ctx_init()
225 	hl_hw_block_mem_init(ctx);  in hl_ctx_init()
228 		ctx->asid = HL_KERNEL_ASID_ID; /* Kernel driver gets ASID 0 */  in hl_ctx_init()
229 		rc = hl_vm_ctx_init(ctx);  in hl_ctx_init()
231 			dev_err(hdev->dev, "Failed to init mem ctx module\n");  in hl_ctx_init()
232 			rc = -ENOMEM;  in hl_ctx_init()
236 		rc = hdev->asic_funcs->ctx_init(ctx);  in hl_ctx_init()
238 			dev_err(hdev->dev, "ctx_init failed\n");  in hl_ctx_init()
242 		ctx->asid = hl_asid_alloc(hdev);  in hl_ctx_init()
243 		if (!ctx->asid) {  in hl_ctx_init()
244 			dev_err(hdev->dev, "No free ASID, failed to create context\n");  in hl_ctx_init()
245 			rc = -ENOMEM;  in hl_ctx_init()
249 		rc = hl_vm_ctx_init(ctx);  in hl_ctx_init()
251 			dev_err(hdev->dev, "Failed to init mem ctx module\n");  in hl_ctx_init()
252 			rc = -ENOMEM;  in hl_ctx_init()
256 		rc = hl_cb_va_pool_init(ctx);  in hl_ctx_init()
258 			dev_err(hdev->dev,  in hl_ctx_init()
263 		rc = hdev->asic_funcs->ctx_init(ctx);  in hl_ctx_init()
265 			dev_err(hdev->dev, "ctx_init failed\n");  in hl_ctx_init()
269 		hl_encaps_sig_mgr_init(&ctx->sig_mgr);  in hl_ctx_init()
271 		mutex_init(&ctx->ts_reg_lock);  in hl_ctx_init()
273 		dev_dbg(hdev->dev, "create user context, comm=\"%s\", asid=%u\n",  in hl_ctx_init()
274 			current->comm, ctx->asid);  in hl_ctx_init()
280 	hl_cb_va_pool_fini(ctx);  in hl_ctx_init()
282 	hl_vm_ctx_fini(ctx);  in hl_ctx_init()
284 	if (ctx->asid != HL_KERNEL_ASID_ID)  in hl_ctx_init()
285 		hl_asid_free(hdev, ctx->asid);  in hl_ctx_init()
287 	hl_hw_block_mem_fini(ctx);  in hl_ctx_init()
288 	kfree(ctx->cs_pending);  in hl_ctx_init()
293 static int hl_ctx_get_unless_zero(struct hl_ctx *ctx)  in hl_ctx_get_unless_zero()  argument
295 	return kref_get_unless_zero(&ctx->refcount);  in hl_ctx_get_unless_zero()
298 void hl_ctx_get(struct hl_ctx *ctx)  in hl_ctx_get()  argument
300 	kref_get(&ctx->refcount);  in hl_ctx_get()
303 int hl_ctx_put(struct hl_ctx *ctx)  in hl_ctx_put()  argument
305 	return kref_put(&ctx->refcount, hl_ctx_do_release);  in hl_ctx_put()
310 	struct hl_ctx *ctx = NULL;  in hl_get_compute_ctx()  local
313 	mutex_lock(&hdev->fpriv_list_lock);  in hl_get_compute_ctx()
315 	list_for_each_entry(hpriv, &hdev->fpriv_list, dev_node) {  in hl_get_compute_ctx()
316 		mutex_lock(&hpriv->ctx_lock);  in hl_get_compute_ctx()
317 		ctx = hpriv->ctx;  in hl_get_compute_ctx()
318 		if (ctx && !hl_ctx_get_unless_zero(ctx))  in hl_get_compute_ctx()
319 			ctx = NULL;  in hl_get_compute_ctx()
320 		mutex_unlock(&hpriv->ctx_lock);  in hl_get_compute_ctx()
328 	mutex_unlock(&hdev->fpriv_list_lock);  in hl_get_compute_ctx()
330 	return ctx;  in hl_get_compute_ctx()
334  * hl_ctx_get_fence_locked - get CS fence under CS lock
336  * @ctx: pointer to the context structure.
344 static struct hl_fence *hl_ctx_get_fence_locked(struct hl_ctx *ctx, u64 seq)  in hl_ctx_get_fence_locked()  argument
346 	struct asic_fixed_properties *asic_prop = &ctx->hdev->asic_prop;  in hl_ctx_get_fence_locked()
349 	if (seq >= ctx->cs_sequence)  in hl_ctx_get_fence_locked()
350 		return ERR_PTR(-EINVAL);  in hl_ctx_get_fence_locked()
352 	if (seq + asic_prop->max_pending_cs < ctx->cs_sequence)  in hl_ctx_get_fence_locked()
355 	fence = ctx->cs_pending[seq & (asic_prop->max_pending_cs - 1)];  in hl_ctx_get_fence_locked()
360 struct hl_fence *hl_ctx_get_fence(struct hl_ctx *ctx, u64 seq)  in hl_ctx_get_fence()  argument
364 	spin_lock(&ctx->cs_lock);  in hl_ctx_get_fence()
366 	fence = hl_ctx_get_fence_locked(ctx, seq);  in hl_ctx_get_fence()
368 	spin_unlock(&ctx->cs_lock);  in hl_ctx_get_fence()
374  * hl_ctx_get_fences - get multiple CS fences under the same CS lock
376  * @ctx: pointer to the context structure.
383 int hl_ctx_get_fences(struct hl_ctx *ctx, u64 *seq_arr,  in hl_ctx_get_fences()  argument
389 	spin_lock(&ctx->cs_lock);  in hl_ctx_get_fences()
394 		*fence = hl_ctx_get_fence_locked(ctx, seq);  in hl_ctx_get_fences()
397 			dev_err(ctx->hdev->dev,  in hl_ctx_get_fences()
405 	spin_unlock(&ctx->cs_lock);  in hl_ctx_get_fences()
414  * hl_ctx_mgr_init - initialize the context manager
423 	mutex_init(&ctx_mgr->lock);  in hl_ctx_mgr_init()
424 	idr_init(&ctx_mgr->handles);  in hl_ctx_mgr_init()
428  * hl_ctx_mgr_fini - finalize the context manager
438 	struct hl_ctx *ctx;  in hl_ctx_mgr_fini()  local
442 	idp = &ctx_mgr->handles;  in hl_ctx_mgr_fini()
444 	idr_for_each_entry(idp, ctx, id)  in hl_ctx_mgr_fini()
445 		kref_put(&ctx->refcount, hl_ctx_do_release);  in hl_ctx_mgr_fini()
447 	idr_destroy(&ctx_mgr->handles);  in hl_ctx_mgr_fini()
448 	mutex_destroy(&ctx_mgr->lock);  in hl_ctx_mgr_fini()